From ff6c024d2bf2947906b82241415ead9e9caeced8 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Thu, 9 Dec 2010 23:48:57 +0900 Subject: [PATCH 01/36] add Kahoku Shinpo News and pet cat blog --- resources/recipes/kahokushinpo.recipe | 32 ++++++++++++++++++++++++ resources/recipes/uninohimitu.recipe | 36 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 resources/recipes/kahokushinpo.recipe create mode 100644 resources/recipes/uninohimitu.recipe diff --git a/resources/recipes/kahokushinpo.recipe b/resources/recipes/kahokushinpo.recipe new file mode 100644 index 0000000000..6e084d83cc --- /dev/null +++ b/resources/recipes/kahokushinpo.recipe @@ -0,0 +1,32 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +www.kahoku.co.jp +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + + +class KahokuShinpoNews(BasicNewsRecipe): + title = u'Kahoku Shinpo News' + __author__ = 'Hiroshi Miura' + oldest_article = 2 + max_articles_per_feed = 20 + description = 'Tohoku regional news paper in Japan' + publisher = 'Kahoku Shinpo Sha' + category = 'news, japan' + language = 'ja' + encoding = 'Shift_JIS' + + + feeds = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')] + + keep_only_tags = [ dict(id="page_title"), + dict(id="news_detail"), + dict(id="bt_title"), + {'class':"photoLeft"}, + dict(id="bt_body") + ] + remove_tags = [ {'class':"button"}] + diff --git a/resources/recipes/uninohimitu.recipe b/resources/recipes/uninohimitu.recipe new file mode 100644 index 0000000000..aac412744c --- /dev/null +++ b/resources/recipes/uninohimitu.recipe @@ -0,0 +1,36 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +http://ameblo.jp/sauta19/ +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class UniNoHimituKichiBlog(BasicNewsRecipe): + title = u'Uni secret base' + __author__ = 'Hiroshi Miura' + oldest_article = 2 + publication_type = 'blog' + max_articles_per_feed = 20 + description = 'Japanese famous Cat blog' + publisher = '' + category = 'cat, pet, japan' + language = 'ja' + encoding = 'utf-8' + + feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/sauta19/rss20.xml')] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'rssad.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds + From 34df6efff9256813718a12174ada30e04311867b Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Fri, 10 Dec 2010 09:50:09 +0900 Subject: [PATCH 02/36] recipe: add popular blog about internet technologies. --- resources/recipes/ajiajin.recipe | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 resources/recipes/ajiajin.recipe diff --git a/resources/recipes/ajiajin.recipe b/resources/recipes/ajiajin.recipe new file mode 100644 index 0000000000..c5f052982b --- /dev/null +++ b/resources/recipes/ajiajin.recipe @@ -0,0 +1,24 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +ajiajin.com/blog +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class AjiajinBlog(BasicNewsRecipe): + title = u'Ajiajin blog' + __author__ = 'Hiroshi Miura' + oldest_article = 5 + publication_type = 'blog' + max_articles_per_feed = 100 + description = 'The next generation internet trends in Japan and Asia' + publisher = '' + category = 'internet, asia, japan' + language = 'en' + encoding = 'utf-8' + + feeds = [(u'blog', u'http://feeds.feedburner.com/Asiajin')] + + From ee5e7abe0b77b6566cf1f215fcac4fe5b49ed697 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sat, 11 Dec 2010 11:30:22 +0900 Subject: [PATCH 03/36] recipe: Nikkei social - fix typo in title and function name --- resources/recipes/nikkei_sub_shakai.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/recipes/nikkei_sub_shakai.recipe b/resources/recipes/nikkei_sub_shakai.recipe index ed86493265..9a53e910e6 100644 --- a/resources/recipes/nikkei_sub_shakai.recipe +++ b/resources/recipes/nikkei_sub_shakai.recipe @@ -10,8 +10,8 @@ import mechanize from calibre.ptempfile import PersistentTemporaryFile -class NikkeiNet_sub_life(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)' +class NikkeiNet_sub_shakai(BasicNewsRecipe): + title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(Social)' __author__ = 'Hiroshi Miura' description = 'News and current market affairs from Japan' cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' From a43274e55a4060bf864ecf1c8f54c64b0c3cee5f Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 12:56:52 +0900 Subject: [PATCH 04/36] recipe: add paper.li recipes --- resources/recipes/paperli.recipe | 58 +++++++++++++++++++++++++ resources/recipes/paperli_topic.recipe | 59 ++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 resources/recipes/paperli.recipe create mode 100644 resources/recipes/paperli_topic.recipe diff --git a/resources/recipes/paperli.recipe b/resources/recipes/paperli.recipe new file mode 100644 index 0000000000..2c99e5dc81 --- /dev/null +++ b/resources/recipes/paperli.recipe @@ -0,0 +1,58 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +paperli +''' + +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +import re, sys + +class paperli(BasicNewsRecipe): +#-------------------please change here ---------------- + paperli_tag = 'osm' + title = u'The # osm Daily - paperli' +#------------------------------------------------------------- + base_url = 'http://paper.li' + index = '/tag/'+paperli_tag+'/~list' + + __author__ = 'Hiroshi Miura' + oldest_article = 7 + max_articles_per_feed = 100 + description = 'paper.li page' + publisher = 'paper.li' + category = 'paper.li' + language = 'en' + encoding = 'utf-8' + remove_javascript = True + timefmt = '[%y/%m/%d]' + + def parse_index(self): + feeds = [] + newsarticles = [] + topic = 'HEADLINE' + + #for pages + page = self.index + while True: + soup = self.index_to_soup(''.join([self.base_url,page])) + for itt in soup.findAll('div',attrs={'class':'yui-u'}): + itema = itt.find('a',href=True,attrs={'class':'ts'}) + if itema is not None: + itemd = itt.find('div',text=True, attrs={'class':'text'}) + newsarticles.append({ + 'title' :itema.string + ,'date' :strftime(self.timefmt) + ,'url' :itema['href'] + ,'description':itemd.string + }) + + nextpage = soup.find('div',attrs={'class':'pagination_top'}).find('li', attrs={'class':'next'}) + if nextpage is not None: + page = nextpage.find('a', href=True)['href'] + else: + break + + feeds.append((topic, newsarticles)) + return feeds + diff --git a/resources/recipes/paperli_topic.recipe b/resources/recipes/paperli_topic.recipe new file mode 100644 index 0000000000..3906af362f --- /dev/null +++ b/resources/recipes/paperli_topic.recipe @@ -0,0 +1,59 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +paperli +''' + +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +import re + +class paperli_topics(BasicNewsRecipe): +#-------------------please change here ---------------- + paperli_tag = 'wikileaks' + title = u'The # wikileaks Daily - paperli' +#------------------------------------------------------------- + __author__ = 'Hiroshi Miura' + oldest_article = 7 + max_articles_per_feed = 100 + description = 'paper.li page about '+ paperli_tag + publisher = 'paper.li' + category = 'paper.li' + language = 'en' + encoding = 'utf-8' + remove_javascript = True + masthead_title = u'The '+ paperli_tag +' Daily' + timefmt = '[%y/%m/%d]' + base_url = 'http://paper.li' + index = base_url+'/tag/'+paperli_tag + + + def parse_index(self): + + # get topics + topics = [] + soup = self.index_to_soup(self.index) + topics_lists = soup.find('div',attrs={'class':'paper-nav-bottom'}) + for item in topics_lists.findAll('li', attrs={'class':""}): + itema = item.find('a',href=True) + topics.append({'title': itema.string, 'url': itema['href']}) + + #get feeds + feeds = [] + for topic in topics: + newsarticles = [] + soup = self.index_to_soup(''.join([self.base_url, topic['url'] ])) + topstories = soup.findAll('div',attrs={'class':'yui-u'}) + for itt in topstories: + itema = itt.find('a',href=True,attrs={'class':'ts'}) + if itema is not None: + itemd = itt.find('div',text=True, attrs={'class':'text'}) + newsarticles.append({ + 'title' :itema.string + ,'date' :strftime(self.timefmt) + ,'url' :itema['href'] + ,'description':itemd.string + }) + feeds.append((topic['title'], newsarticles)) + return feeds + From 1efd975625c1f32e52722f7ab18e3f099496c274 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 12:58:32 +0900 Subject: [PATCH 05/36] recipe: fix kahoku shinpo --- resources/recipes/kahokushinpo.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/recipes/kahokushinpo.recipe b/resources/recipes/kahokushinpo.recipe index 6e084d83cc..172014d3a0 100644 --- a/resources/recipes/kahokushinpo.recipe +++ b/resources/recipes/kahokushinpo.recipe @@ -9,7 +9,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class KahokuShinpoNews(BasicNewsRecipe): - title = u'Kahoku Shinpo News' + title = u'\u6cb3\u5317\u65b0\u5831' __author__ = 'Hiroshi Miura' oldest_article = 2 max_articles_per_feed = 20 @@ -18,7 +18,7 @@ class KahokuShinpoNews(BasicNewsRecipe): category = 'news, japan' language = 'ja' encoding = 'Shift_JIS' - + no_stylesheets = True feeds = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')] From d18bef33e11c20be510339e9ffe7bca665ff6dde Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 22:28:55 +0900 Subject: [PATCH 06/36] recipe: add national geographic news - national geographic Japan - national geographic News --- resources/recipes/nationalgeographic.recipe | 38 +++++++++++++++++++ resources/recipes/nationalgeographicjp.recipe | 20 ++++++++++ 2 files changed, 58 insertions(+) create mode 100644 resources/recipes/nationalgeographic.recipe create mode 100644 resources/recipes/nationalgeographicjp.recipe diff --git a/resources/recipes/nationalgeographic.recipe b/resources/recipes/nationalgeographic.recipe new file mode 100644 index 0000000000..b540f9b044 --- /dev/null +++ b/resources/recipes/nationalgeographic.recipe @@ -0,0 +1,38 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +nationalgeographic.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class NationalGeographicNews(BasicNewsRecipe): + title = u'National Geographic News' + oldest_article = 7 + max_articles_per_feed = 100 + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + feeds = [(u'news', u'http://feeds.nationalgeographic.com/ng/News/News_Main')] + + remove_tags_before = dict(id='page_head') + remove_tags_after = [dict(id='social_buttons'),{'class':'aside'}] + remove_tags = [ + {'class':'hidden'} + + ] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'ads\.pheedo\.com', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds diff --git a/resources/recipes/nationalgeographicjp.recipe b/resources/recipes/nationalgeographicjp.recipe new file mode 100644 index 0000000000..5798acb102 --- /dev/null +++ b/resources/recipes/nationalgeographicjp.recipe @@ -0,0 +1,20 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +nationalgeographic.co.jp +''' + +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class NationalGeoJp(BasicNewsRecipe): + title = u'\u30ca\u30b7\u30e7\u30ca\u30eb\u30fb\u30b8\u30aa\u30b0\u30e9\u30d5\u30a3\u30c3\u30af\u30cb\u30e5\u30fc\u30b9' + oldest_article = 7 + max_articles_per_feed = 100 + no_stylesheets = True + + feeds = [(u'news', u'http://www.nationalgeographic.co.jp/news/rss.php')] + + def print_version(self, url): + return re.sub(r'news_article.php','news_printer_friendly.php', url) + From c3bbe2cc8659db1c13bf4f001c09bb3e3f658145 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 22:46:55 +0900 Subject: [PATCH 07/36] recipe: add dog blog in Japanese --- resources/recipes/chouchoublog.recipe | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 resources/recipes/chouchoublog.recipe diff --git a/resources/recipes/chouchoublog.recipe b/resources/recipes/chouchoublog.recipe new file mode 100644 index 0000000000..8c953deef0 --- /dev/null +++ b/resources/recipes/chouchoublog.recipe @@ -0,0 +1,37 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +http://ameblo.jp/ +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class SakuraBlog(BasicNewsRecipe): + title = u'chou chou blog' + __author__ = 'Hiroshi Miura' + oldest_article = 4 + publication_type = 'blog' + max_articles_per_feed = 20 + description = 'Japanese popular dog blog' + publisher = '' + category = 'dog, pet, japan' + language = 'ja' + encoding = 'utf-8' + use_embedded_content = True + + feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/chouchou1218/rss20.xml')] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'rssad.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds + From 9ea944ff143315d88dced7a5aae538566cb1a730 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:35:12 -0700 Subject: [PATCH 08/36] Conversion pipeline: Add an option to set the minimum line height of all elemnts as a percentage of the computed font size. By default, calibre now sets the line height to 120% of the computed font size. --- src/calibre/ebooks/conversion/cli.py | 2 +- src/calibre/ebooks/conversion/plumber.py | 25 ++++++++-- src/calibre/ebooks/oeb/stylizer.py | 6 +-- src/calibre/ebooks/oeb/transforms/flatcss.py | 11 +++++ src/calibre/gui2/convert/look_and_feel.py | 2 +- src/calibre/gui2/convert/look_and_feel.ui | 51 ++++++++++++++------ 6 files changed, 74 insertions(+), 23 deletions(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index 62a941142b..3178fe1b43 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -120,7 +120,7 @@ def add_pipeline_options(parser, plumber): [ 'base_font_size', 'disable_font_rescaling', 'font_size_mapping', - 'line_height', + 'line_height', 'minimum_line_height', 'linearize_tables', 'extra_css', 'smarten_punctuation', 'margin_top', 'margin_left', 'margin_right', diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 9a863d7e66..f5beba375d 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -160,13 +160,30 @@ OptionRecommendation(name='disable_font_rescaling', ) ), +OptionRecommendation(name='minimum_line_height', + recommended_value=120.0, level=OptionRecommendation.LOW, + help=_( + 'The minimum line height, as a percentage of the element\'s ' + 'calculated font size. calibre will ensure that every element ' + 'has a line height of at least this setting, irrespective of ' + 'what the input document specifies. Set to zero to disable. ' + 'Default is 120%. Use this setting in preference to ' + 'the direct line height specification, unless you know what ' + 'you are doing. For example, you can achieve "double spaced" ' + 'text by setting this to 240.' + ) + ), + OptionRecommendation(name='line_height', recommended_value=0, level=OptionRecommendation.LOW, - help=_('The line height in pts. Controls spacing between consecutive ' - 'lines of text. By default no line height manipulation is ' - 'performed.' - ) + help=_( + 'The line height in pts. Controls spacing between consecutive ' + 'lines of text. Only applies to elements that do not define ' + 'their own line height. In most cases, the minimum line height ' + 'option is more useful. ' + 'By default no line height manipulation is performed.' + ) ), OptionRecommendation(name='linearize_tables', diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 6c0c384eb3..616cd3b800 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -633,12 +633,12 @@ class Style(object): parent = self._getparent() if 'line-height' in self._style: lineh = self._style['line-height'] + if lineh == 'normal': + lineh = '1.2' try: - float(lineh) + result = float(lineh) * self.fontSize except ValueError: result = self._unit_convert(lineh, base=self.fontSize) - else: - result = float(lineh) * self.fontSize elif parent is not None: # TODO: proper inheritance result = parent.lineHeight diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 7b83421097..653aa4533b 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -245,6 +245,8 @@ class CSSFlattener(object): del node.attrib['bgcolor'] if cssdict.get('font-weight', '').lower() == 'medium': cssdict['font-weight'] = 'normal' # ADE chokes on font-weight medium + + fsize = font_size if not self.context.disable_font_rescaling: _sbase = self.sbase if self.sbase is not None else \ self.context.source.fbase @@ -258,6 +260,14 @@ class CSSFlattener(object): fsize = self.fmap[font_size] cssdict['font-size'] = "%0.5fem" % (fsize / psize) psize = fsize + + try: + minlh = self.context.minimum_line_height / 100. + if style['line-height'] < minlh * fsize: + cssdict['line-height'] = str(minlh) + except: + self.oeb.logger.exception('Failed to set minimum line-height') + if cssdict: if self.lineh and self.fbase and tag != 'body': self.clean_edges(cssdict, style, psize) @@ -290,6 +300,7 @@ class CSSFlattener(object): lineh = self.lineh / psize cssdict['line-height'] = "%0.5fem" % lineh + if (self.context.remove_paragraph_spacing or self.context.insert_blank_line) and tag in ('p', 'div'): if item_id != 'calibre_jacket' or self.context.output_profile.name == 'Kindle': diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index ec3f0b944d..98b9cb8155 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -21,7 +21,7 @@ class LookAndFeelWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, ['change_justification', 'extra_css', 'base_font_size', - 'font_size_mapping', 'line_height', + 'font_size_mapping', 'line_height', 'minimum_line_height', 'linearize_tables', 'smarten_punctuation', 'disable_font_rescaling', 'insert_blank_line', 'remove_paragraph_spacing', 'remove_paragraph_spacing_indent_size','input_encoding', diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index c683300854..367233e2c0 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -97,7 +97,7 @@ - + Line &height: @@ -107,7 +107,7 @@ - + pt @@ -117,7 +117,7 @@ - + Input character &encoding: @@ -127,17 +127,17 @@ - + - + Remove &spacing between paragraphs - + @@ -164,21 +164,21 @@ - + Text justification: - + &Linearize tables - + Extra &CSS @@ -190,37 +190,60 @@ - + - + &Transliterate unicode characters to ASCII - + Insert &blank line - + Keep &ligatures - + Smarten &punctuation + + + + Minimum &line height: + + + opt_minimum_line_height + + + + + + + % + + + 1 + + + 900.000000000000000 + + + From e177f043c0527d88807fe01ca2d31329bddff660 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:36:47 -0700 Subject: [PATCH 09/36] TXT Input: Use a nicer name for the generated html file --- src/calibre/ebooks/txt/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index b444bf1cf4..e82b980009 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,7 +77,7 @@ class TXTInput(InputFormatPlugin): base = os.getcwdu() if hasattr(stream, 'name'): base = os.path.dirname(stream.name) - htmlfile = open(os.path.join(base, 'temp_calibre_txt_input_to_html.html'), + htmlfile = open(os.path.join(base, 'index.html'), 'wb') htmlfile.write(html.encode('utf-8')) htmlfile.close() From 2dd9b3f128bfa7f4f5cbe20744c0426af4895284 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:45:55 -0700 Subject: [PATCH 10/36] ... --- src/calibre/ebooks/txt/input.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e82b980009..44b98304ea 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,10 +77,14 @@ class TXTInput(InputFormatPlugin): base = os.getcwdu() if hasattr(stream, 'name'): base = os.path.dirname(stream.name) - htmlfile = open(os.path.join(base, 'index.html'), - 'wb') - htmlfile.write(html.encode('utf-8')) - htmlfile.close() + fname = os.path.join(base, 'index.html') + c = 0 + while os.path.exists(fname): + c += 1 + fname = 'index%d.html'%c + htmlfile = open(fname, 'wb') + with htmlfile: + htmlfile.write(html.encode('utf-8')) cwd = os.getcwdu() odi = options.debug_pipeline options.debug_pipeline = None From 4a8551962fdbe65ba4c77a7e08f0ae1baa4f948b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 11:07:31 -0700 Subject: [PATCH 11/36] Edit metadata dialog: clean up handling of cover fetch thread --- src/calibre/gui2/dialogs/metadata_single.py | 43 ++++++++++----------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 3205b1d23c..d9bb1c2a33 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -240,37 +240,39 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.cover_fetcher = CoverFetcher(None, None, isbn, self.timeout, title, author) self.cover_fetcher.start() - self._hangcheck = QTimer(self) - self._hangcheck.timeout.connect(self.hangcheck, - type=Qt.QueuedConnection) self.cf_start_time = time.time() self.pi.start(_('Downloading cover...')) - self._hangcheck.start(100) + QTimer.singleShot(100, self.hangcheck) def hangcheck(self): - if self.cover_fetcher.is_alive() and \ - time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: + cf = self.cover_fetcher + if cf is None: + # Called after dialog closed + return + + if cf.is_alive() and \ + time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: + QTimer.singleShot(100, self.hangcheck) return - self._hangcheck.stop() try: - if self.cover_fetcher.is_alive(): + if cf.is_alive(): error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
')+ _('The download timed out.')).exec_() return - if self.cover_fetcher.needs_isbn: + if cf.needs_isbn: error_dialog(self, _('Cannot fetch cover'), _('Could not find cover for this book. Try ' 'specifying the ISBN first.')).exec_() return - if self.cover_fetcher.exception is not None: - err = self.cover_fetcher.exception + if cf.exception is not None: + err = cf.exception error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
')+unicode(err)).exec_() return - if self.cover_fetcher.errors and self.cover_fetcher.cover_data is None: - details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in self.cover_fetcher.errors]) + if cf.errors and cf.cover_data is None: + details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in cf.errors]) error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
') + _('For the error message from each cover source, ' @@ -278,7 +280,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): return pix = QPixmap() - pix.loadFromData(self.cover_fetcher.cover_data) + pix.loadFromData(cf.cover_data) if pix.isNull(): error_dialog(self, _('Bad cover'), _('The cover is not a valid picture')).exec_() @@ -287,7 +289,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.update_cover_tooltip() self.cover_changed = True self.cpixmap = pix - self.cover_data = self.cover_fetcher.cover_data + self.cover_data = cf.cover_data finally: self.fetch_cover_button.setEnabled(True) self.unsetCursor() @@ -438,6 +440,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): def __init__(self, window, row, db, prev=None, next_=None): ResizableDialog.__init__(self, window) + self.cover_fetcher = None self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) self.cancel_all = False base = unicode(self.author_sort.toolTip()) @@ -828,10 +831,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.accept() def accept(self): - cf = getattr(self, 'cover_fetcher', None) - if cf is not None and hasattr(cf, 'terminate'): - cf.terminate() - cf.wait() try: if self.formats_changed: self.sync_formats() @@ -888,14 +887,12 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): show=True) raise self.save_state() + self.cover_fetcher = None QDialog.accept(self) def reject(self, *args): - cf = getattr(self, 'cover_fetcher', None) - if cf is not None and hasattr(cf, 'terminate'): - cf.terminate() - cf.wait() self.save_state() + self.cover_fetcher = None QDialog.reject(self, *args) def read_state(self): From 0e70bb8b592afbb950b05af6a99adc2afe5682d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 11:45:19 -0700 Subject: [PATCH 12/36] Preferences: Add tooltips to buddy labels as well. Fixes #7873 (No tooltip for some format conversion fields) --- src/calibre/gui2/convert/__init__.py | 17 ++++++++++++++++- src/calibre/gui2/preferences/main.py | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index 0c3a8b5a4e..c1efe5b9af 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -10,7 +10,7 @@ import textwrap from functools import partial from PyQt4.Qt import QWidget, QSpinBox, QDoubleSpinBox, QLineEdit, QTextEdit, \ - QCheckBox, QComboBox, Qt, QIcon, pyqtSignal + QCheckBox, QComboBox, Qt, QIcon, pyqtSignal, QLabel from calibre.customize.conversion import OptionRecommendation from calibre.ebooks.conversion.config import load_defaults, \ @@ -81,6 +81,21 @@ class Widget(QWidget): self.apply_recommendations(defaults) self.setup_help(get_help) + def process_child(child): + for g in child.children(): + if isinstance(g, QLabel): + buddy = g.buddy() + if buddy is not None and hasattr(buddy, '_help'): + g._help = buddy._help + htext = unicode(buddy.toolTip()).strip() + g.setToolTip(htext) + g.setWhatsThis(htext) + g.__class__.enterEvent = lambda obj, event: self.set_help(getattr(obj, '_help', obj.toolTip())) + else: + process_child(g) + process_child(self) + + def restore_defaults(self, get_option): defaults = GuiRecommendations() defaults.merge_recommendations(get_option, OptionRecommendation.LOW, diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index fc01a33cf6..f7d49427c8 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -251,10 +251,28 @@ class Preferences(QMainWindow): self.close() self.run_wizard_requested.emit() + def set_tooltips_for_labels(self): + + def process_child(child): + for g in child.children(): + if isinstance(g, QLabel): + buddy = g.buddy() + if buddy is not None and hasattr(buddy, 'toolTip'): + htext = unicode(buddy.toolTip()).strip() + etext = unicode(g.toolTip()).strip() + if htext and not etext: + g.setToolTip(htext) + g.setWhatsThis(htext) + else: + process_child(g) + + process_child(self.showing_widget) + def show_plugin(self, plugin): self.showing_widget = plugin.create_widget(self.scroll_area) self.showing_widget.genesis(self.gui) self.showing_widget.initialize() + self.set_tooltips_for_labels() self.scroll_area.setWidget(self.showing_widget) self.stack.setCurrentIndex(1) self.showing_widget.show() From 524e9f4def3e41d8f00644262db1c4cffdab3984 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:09:43 -0700 Subject: [PATCH 13/36] Implement #7877 (margin option in epub to mobi conversions) --- src/calibre/ebooks/mobi/mobiml.py | 2 +- src/calibre/ebooks/mobi/output.py | 6 ++++++ src/calibre/gui2/convert/mobi_output.py | 1 + src/calibre/gui2/convert/mobi_output.ui | 11 +++++++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 8d20179250..001cf2c1e9 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -184,7 +184,7 @@ class MobiMLizer(object): para.attrib['value'] = str(istates[-2].list_num) elif tag in NESTABLE_TAGS and istate.rendered: para = wrapper = bstate.nested[-1] - elif left > 0 and indent >= 0: + elif not self.opts.mobi_ignore_margins and left > 0 and indent >= 0: ems = self.profile.mobi_ems_per_blockquote para = wrapper = etree.SubElement(parent, XHTML('blockquote')) para = wrapper diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 4159c6dd40..a6f6c52b7f 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -39,6 +39,12 @@ class MOBIOutput(OutputFormatPlugin): OptionRecommendation(name='personal_doc', recommended_value='[PDOC]', help=_('Tag marking book to be filed with Personal Docs') ), + OptionRecommendation(name='mobi_ignore_margins', + recommended_value=False, + help=_('Ignore margins in the input document. If False, then ' + 'the MOBI output plugin will try to convert margins specified' + ' in the input document, otherwise it will ignore them.') + ), ]) def check_for_periodical(self): diff --git a/src/calibre/gui2/convert/mobi_output.py b/src/calibre/gui2/convert/mobi_output.py index 23c0b30253..14aca24db5 100644 --- a/src/calibre/gui2/convert/mobi_output.py +++ b/src/calibre/gui2/convert/mobi_output.py @@ -25,6 +25,7 @@ class PluginWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, ['prefer_author_sort', 'rescale_images', 'toc_title', + 'mobi_ignore_margins', 'dont_compress', 'no_inline_toc', 'masthead_font','personal_doc'] ) self.db, self.book_id = db, book_id diff --git a/src/calibre/gui2/convert/mobi_output.ui b/src/calibre/gui2/convert/mobi_output.ui index 176ce681c0..e9eab45e1a 100644 --- a/src/calibre/gui2/convert/mobi_output.ui +++ b/src/calibre/gui2/convert/mobi_output.ui @@ -55,7 +55,7 @@ - + Kindle options @@ -101,7 +101,7 @@ - + Qt::Vertical @@ -114,6 +114,13 @@ + + + + Ignore &margins + + + From e56e2c7f45580352d71ab93a8337fdd3678ee9fc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:24:26 -0700 Subject: [PATCH 14/36] Improved recipe for Dilbert --- resources/recipes/dilbert.recipe | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/resources/recipes/dilbert.recipe b/resources/recipes/dilbert.recipe index 82966b1d15..2c3268da2f 100644 --- a/resources/recipes/dilbert.recipe +++ b/resources/recipes/dilbert.recipe @@ -3,15 +3,16 @@ __copyright__ = '2009, Darko Miletic ' ''' http://www.dilbert.com ''' -import re from calibre.web.feeds.recipes import BasicNewsRecipe +import re -class DosisDiarias(BasicNewsRecipe): +class DilbertBig(BasicNewsRecipe): title = 'Dilbert' - __author__ = 'Darko Miletic' + __author__ = 'Darko Miletic and Starson17' description = 'Dilbert' - oldest_article = 5 + reverse_article_order = True + oldest_article = 15 max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = True @@ -29,20 +30,23 @@ class DosisDiarias(BasicNewsRecipe): feeds = [(u'Dilbert', u'http://feeds.dilbert.com/DilbertDailyStrip' )] - preprocess_regexps = [ - (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), - lambda match: 'strip.zoom.gif') - ] - - def get_article_url(self, article): return article.get('feedburner_origlink', None) + preprocess_regexps = [ + (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), lambda match: 'strip.zoom.gif') + ] + def preprocess_html(self, soup): for tag in soup.findAll(name='a'): if tag['href'].find('http://feedads') >= 0: tag.extract() return soup - - + extra_css = ''' + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + img {max-width:100%; min-width:100%;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} + ''' From 86ee4489d4a99c99930c646dbcc273b920b7a821 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:55:53 -0700 Subject: [PATCH 15/36] Fix #7851 (some meta tags failed since V0.7.25) --- src/calibre/ebooks/html/input.py | 12 +++--------- src/calibre/ebooks/metadata/html.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 059aeca324..6f875ae803 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -314,6 +314,8 @@ class HTMLInput(InputFormatPlugin): rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, \ xpath from calibre import guess_type + from calibre.ebooks.oeb.transforms.metadata import \ + meta_info_to_oeb_metadata import cssutils self.OEB_STYLES = OEB_STYLES oeb = create_oebbook(log, None, opts, self, @@ -321,15 +323,7 @@ class HTMLInput(InputFormatPlugin): self.oeb = oeb metadata = oeb.metadata - if mi.title: - metadata.add('title', mi.title) - if mi.authors: - for a in mi.authors: - metadata.add('creator', a, attrib={'role':'aut'}) - if mi.publisher: - metadata.add('publisher', mi.publisher) - if mi.isbn: - metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'}) + meta_info_to_oeb_metadata(mi, metadata, log) if not metadata.language: oeb.logger.warn(u'Language not specified') metadata.add('language', get_lang().replace('_', '-')) diff --git a/src/calibre/ebooks/metadata/html.py b/src/calibre/ebooks/metadata/html.py index f4eaa7cc61..fd42b2882f 100644 --- a/src/calibre/ebooks/metadata/html.py +++ b/src/calibre/ebooks/metadata/html.py @@ -170,7 +170,27 @@ def get_metadata_(src, encoding=None): if match: series = match.group(1) if series: + pat = re.compile(r'\[([.0-9]+)\]') + match = pat.search(series) + series_index = None + if match is not None: + try: + series_index = float(match.group(1)) + except: + pass + series = series.replace(match.group(), '').strip() + mi.series = ent_pat.sub(entity_to_unicode, series) + if series_index is None: + pat = get_meta_regexp_("Seriesnumber") + match = pat.search(src) + if match: + try: + series_index = float(match.group(1)) + except: + pass + if series_index is not None: + mi.series_index = series_index # RATING rating = None From ae8764d4563e2e88f3f56976f446e9198d4386a8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Dec 2010 12:33:33 +0000 Subject: [PATCH 16/36] Enhancement #7881 - keep manage tags editor positioned near last delete. --- src/calibre/gui2/dialogs/tag_list_editor.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py index a7d6fe03e7..ced0e9a505 100644 --- a/src/calibre/gui2/dialogs/tag_list_editor.py +++ b/src/calibre/gui2/dialogs/tag_list_editor.py @@ -105,9 +105,13 @@ class TagListEditor(QDialog, Ui_TagListEditor): if not question_dialog(self, _('Are your sure?'), '

'+_('Are you certain you want to delete the following items?')+'
'+ct): return - + row = self.available_tags.row(deletes[0]) for item in deletes: (id,ign) = item.data(Qt.UserRole).toInt() self.to_delete.append(id) self.available_tags.takeItem(self.available_tags.row(item)) + if row >= self.available_tags.count(): + row = self.available_tags.count() - 1 + if row >= 0: + self.available_tags.scrollToItem(self.available_tags.item(row)) From a110eb19ddd82eac08e63b5bf98539d52e578bf3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Dec 2010 15:08:05 +0000 Subject: [PATCH 17/36] Fix bug in sorting using icu sort_key --- src/calibre/gui2/dialogs/tag_categories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index 60092e4bd2..7573f04012 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -145,7 +145,7 @@ class TagCategories(QDialog, Ui_TagCategories): index = self.all_items[node.data(Qt.UserRole).toPyObject()].index if index not in self.applied_items: self.applied_items.append(index) - self.applied_items.sort(key=lambda x:sort_key(self.all_items[x])) + self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name)) self.display_filtered_categories(None) def unapply_tags(self, node=None): From 881bfef0f2ca3298bd85ace291d90fd974d8c2a8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Dec 2010 15:12:53 +0000 Subject: [PATCH 18/36] Avoid doing tag_view.recount() when the tags pane is not visible --- src/calibre/gui2/init.py | 2 ++ src/calibre/gui2/tag_view.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 27a6a2352a..fc70f0579d 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -123,6 +123,8 @@ class Stack(QStackedWidget): # {{{ _('Tag Browser'), I('tags.png'), parent=parent, side_index=0, initial_side_size=200, shortcut=_('Shift+Alt+T')) + parent.tb_splitter.state_changed.connect( + self.tb_widget.set_pane_is_visible, Qt.QueuedConnection) parent.tb_splitter.addWidget(self.tb_widget) parent.tb_splitter.addWidget(parent.cb_splitter) parent.tb_splitter.setCollapsible(parent.tb_splitter.other_index, False) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 2ede698c85..d6c0156f13 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -87,6 +87,13 @@ class TagsView(QTreeView): # {{{ self.setDragDropMode(self.DropOnly) self.setDropIndicatorShown(True) self.setAutoExpandDelay(500) + self.pane_is_visible = False + + def set_pane_is_visible(self, to_what): + pv = self.pane_is_visible + self.pane_is_visible = to_what + if to_what and not pv: + self.recount() def set_database(self, db, tag_match, sort_by): self.hidden_categories = config['tag_browser_hidden_categories'] @@ -300,7 +307,7 @@ class TagsView(QTreeView): # {{{ return self.isExpanded(idx) def recount(self, *args): - if self.disable_recounting: + if self.disable_recounting or not self.pane_is_visible: return self.refresh_signal_processed = True ci = self.currentIndex() @@ -969,6 +976,7 @@ class TagBrowserWidget(QWidget): # {{{ self._layout.setContentsMargins(0,0,0,0) parent.tags_view = TagsView(parent) + self.tags_view = parent.tags_view self._layout.addWidget(parent.tags_view) parent.sort_by = QComboBox(parent) @@ -998,6 +1006,9 @@ class TagBrowserWidget(QWidget): # {{{ _('Add your own categories to the Tag Browser')) parent.edit_categories.setStatusTip(parent.edit_categories.toolTip()) + def set_pane_is_visible(self, to_what): + self.tags_view.set_pane_is_visible(to_what) + # }}} From c260cb052b7cea5b5e43be29bfe57f3d384f2d21 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 13 Dec 2010 19:11:41 +0000 Subject: [PATCH 19/36] Fix #7888: empty tags list throws exception in save_to_disk, converting the result to 'tags' --- src/calibre/library/save_to_disk.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index af57d563ac..62a2e28e27 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -119,10 +119,8 @@ class SafeFormat(TemplateFormatter): try: b = self.book.get_user_metadata(key, False) except: - if DEBUG: - traceback.print_exc() + traceback.print_exc() b = None - if b is not None and b['datatype'] == 'composite': if key in self.composite_values: return self.composite_values[key] @@ -135,8 +133,7 @@ class SafeFormat(TemplateFormatter): return val.replace('/', '_').replace('\\', '_') return '' except: - if DEBUG: - traceback.print_exc() + traceback.print_exc() return key def get_components(template, mi, id, timefmt='%b %Y', length=250, @@ -155,6 +152,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, format_args['tags'] = mi.format_tags() if format_args['tags'].startswith('/'): format_args['tags'] = format_args['tags'][1:] + else: + format_args['tags'] = '' if mi.series: format_args['series'] = tsfmt(mi.series) if mi.series_index is not None: From be0f3b096a019d35ff35f390472cb254da8c1e23 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 13:29:07 -0700 Subject: [PATCH 20/36] Fix a regression in 0.7.33 that broke updating covers in ebook files when saving to disk. Fixes #7886 (Possible bug in setting new cover) --- src/calibre/library/save_to_disk.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 62a2e28e27..7090a2afa8 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -253,6 +253,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, if not os.path.exists(dirpath): raise + ocover = mi.cover if opts.save_cover and cover and os.access(cover, os.R_OK): with open(base_path+'.jpg', 'wb') as f: with open(cover, 'rb') as s: @@ -266,6 +267,8 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, with open(base_path+'.opf', 'wb') as f: f.write(opf) + mi.cover = ocover + written = False for fmt in formats: global plugboard_save_to_disk_value, plugboard_any_format_value From 3a1fd7af5665ddcfb8523ae460359703e7779ac8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 15:06:34 -0700 Subject: [PATCH 21/36] Bulk metadata edit: Add options to delete cover/generate default cover. Fixes #7885 (Bulk remove covers) --- src/calibre/gui2/dialogs/metadata_bulk.py | 26 +++++++++++++++-- src/calibre/gui2/dialogs/metadata_bulk.ui | 35 ++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index a640c50fb8..e0f1f83c73 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -102,7 +102,7 @@ class MyBlockingBusy(QDialog): remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, \ do_autonumber, do_remove_format, remove_format, do_swap_ta, \ do_remove_conv, do_auto_author, series, do_series_restart, \ - series_start_value, do_title_case, clear_series = self.args + series_start_value, do_title_case, cover_action, clear_series = self.args # first loop: do author and title. These will commit at the end of each @@ -129,6 +129,23 @@ class MyBlockingBusy(QDialog): self.db.set_title(id, titlecase(title), notify=False) if au: self.db.set_authors(id, string_to_authors(au), notify=False) + if cover_action == 'remove': + self.db.remove_cover(id) + elif cover_action == 'generate': + from calibre.ebooks import calibre_cover + from calibre.ebooks.metadata import fmt_sidx + from calibre.gui2 import config + mi = self.db.get_metadata(id, index_is_id=True) + series_string = None + if mi.series: + series_string = _('Book %s of %s')%( + fmt_sidx(mi.series_index, + use_roman=config['use_roman_numerals_for_series_number']), + mi.series) + + cdata = calibre_cover(mi.title, mi.format_field('authors')[-1], + series_string=series_string) + self.db.set_cover(id, cdata) elif self.current_phase == 2: # All of these just affect the DB, so we can tolerate a total rollback if do_auto_author: @@ -678,11 +695,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): do_remove_conv = self.remove_conversion_settings.isChecked() do_auto_author = self.auto_author_sort.isChecked() do_title_case = self.change_title_to_title_case.isChecked() + cover_action = None + if self.cover_remove.isChecked(): + cover_action = 'remove' + elif self.cover_generate.isChecked(): + cover_action = 'generate' args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, do_autonumber, do_remove_format, remove_format, do_swap_ta, do_remove_conv, do_auto_author, series, do_series_restart, - series_start_value, do_title_case, clear_series) + series_start_value, do_title_case, cover_action, clear_series) bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') %len(self.ids), args, self.db, self.ids, diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 344bde0fa0..cd644f88ba 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -381,7 +381,7 @@ Future conversion of these books will use the default settings. - + Qt::Vertical @@ -394,6 +394,39 @@ Future conversion of these books will use the default settings. + + + + Change &cover + + + + + + &No change + + + true + + + + + + + &Remove cover + + + + + + + &Generate default cover + + + + + + From 97536e397fc138b581e04ebb1f26c8e41efdf38f Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 13 Dec 2010 18:38:37 -0500 Subject: [PATCH 22/36] FB2 Output: add blank line after paragraphs when insert-blank-line option used. Use instead of CSS because not many readers support CSS in FB2 files. --- src/calibre/ebooks/fb2/fb2ml.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 51bfaa7293..89c12db103 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -73,6 +73,10 @@ class FB2MLizer(object): text = re.sub(r'(?miu)

\s*

', '', text) text = re.sub(r'(?miu)\s+

', '

', text) text = re.sub(r'(?miu)

', '

\n\n

', text) + + if self.opts.insert_blank_line: + text = re.sub(r'(?miu)

', '

', text) + return text def fb2_header(self): From 3df1780251d802d7de47bbf0484a932e0bf945bf Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 13 Dec 2010 18:57:05 -0500 Subject: [PATCH 23/36] FB2 Output: Add support for some 2.1 style tags. --- src/calibre/ebooks/fb2/fb2ml.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 89c12db103..5efc360f1f 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -297,6 +297,18 @@ class FB2MLizer(object): s_out, s_tags = self.handle_simple_tag('emphasis', tag_stack+tags) fb2_out += s_out tags += s_tags + elif tag in ('del', 'strike'): + s_out, s_tags = self.handle_simple_tag('strikethrough', tag_stack+tags) + fb2_out += s_out + tags += s_tags + elif tag == 'sub': + s_out, s_tags = self.handle_simple_tag('sub', tag_stack+tags) + fb2_out += s_out + tags += s_tags + elif tag == 'sup': + s_out, s_tags = self.handle_simple_tag('sup', tag_stack+tags) + fb2_out += s_out + tags += s_tags # Processes style information. if style['font-style'] == 'italic': @@ -307,6 +319,10 @@ class FB2MLizer(object): s_out, s_tags = self.handle_simple_tag('strong', tag_stack+tags) fb2_out += s_out tags += s_tags + elif style['text-decoration'] == 'line-through': + s_out, s_tags = self.handle_simple_tag('strikethrough', tag_stack+tags) + fb2_out += s_out + tags += s_tags # Process element text. if hasattr(elem_tree, 'text') and elem_tree.text: From acbbc16bf3af727e317efd87d9d4b146950747ea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 17:07:00 -0700 Subject: [PATCH 24/36] Infrastructure to detect mem leaks in the edit metadata dialog --- src/calibre/gui2/actions/edit_metadata.py | 8 ++- src/calibre/gui2/dialogs/metadata_single.py | 69 ++++++++++++++++++++- src/calibre/utils/mem.py | 55 ++++++++++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 src/calibre/utils/mem.py diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 60a943ccb9..11949632e9 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -154,15 +154,17 @@ class EditMetadataAction(InterfaceAction): d.view_format.connect(lambda fmt:self.gui.iactions['View'].view_format(row_list[current_row], fmt)) - if d.exec_() != d.Accepted: - d.view_format.disconnect() + ret = d.exec_() + d.break_cycles() + if ret != d.Accepted: break - d.view_format.disconnect() + changed.add(d.id) if d.row_delta == 0: break current_row += d.row_delta + if changed: self.gui.library_view.model().refresh_ids(list(changed)) current = self.gui.library_view.currentIndex() diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index d9bb1c2a33..9e3a8c7eda 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -293,7 +293,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): finally: self.fetch_cover_button.setEnabled(True) self.unsetCursor() - self.pi.stop() + if self.pi is not None: + self.pi.stop() # }}} @@ -442,7 +443,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): ResizableDialog.__init__(self, window) self.cover_fetcher = None self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) - self.cancel_all = False base = unicode(self.author_sort.toolTip()) self.ok_aus_tooltip = '

' + textwrap.fill(base+'

'+ _(' The green color indicates that the current ' @@ -573,7 +573,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): QObject.connect(self.series, SIGNAL('editTextChanged(QString)'), self.enable_series_index) self.series.lineEdit().editingFinished.connect(self.increment_series_index) - self.show() pm = QPixmap() if cover: pm.loadFromData(cover) @@ -593,6 +592,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.original_author = unicode(self.authors.text()).strip() self.original_title = unicode(self.title.text()).strip() + self.show() + def create_custom_column_editors(self): w = self.central_widget.widget(1) layout = w.layout() @@ -907,3 +908,65 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): dynamic.set('metasingle_window_geometry', bytes(self.saveGeometry())) dynamic.set('metasingle_splitter_state', bytes(self.splitter.saveState())) + + def break_cycles(self): + try: + self.view_format.disconnect() + except: + pass # Fails if view format was never connected + self.view_format = None + self.db = None + self.pi = None + self.cover_data = self.cpixmap = None + +if __name__ == '__main__': + from calibre.library import db + from PyQt4.Qt import QApplication + from calibre.utils.mem import memory + import gc + + + app = QApplication([]) + db = db() + + # Initialize all Qt Objects once + d = MetadataSingleDialog(None, 4, db) + d.break_cycles() + d.reject() + del d + + for i in range(3): + gc.collect() + before = memory() + + gc.collect() + d = MetadataSingleDialog(None, 4, db) + d.break_cycles() + d.reject() + del d + + for i in range(3): + gc.collect() + print 'Used memory:', memory(before)/1024.**2, 'MB' + gc.collect() + + ''' + nmap, omap = {}, {} + for x in objects: + omap[id(x)] = x + for x in nobjects: + nmap[id(x)] = x + + new_ids = set(nmap.keys()) - set(omap.keys()) + print "New ids:", len(new_ids) + for i in new_ids: + o = nmap[i] + if o is objects: + continue + print repr(o)[:1050] + refs = gc.get_referrers(o) + for r in refs: + if r is objects or r is nobjects: + continue + print '\t', r + ''' diff --git a/src/calibre/utils/mem.py b/src/calibre/utils/mem.py new file mode 100644 index 0000000000..f48aec34c6 --- /dev/null +++ b/src/calibre/utils/mem.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +## {{{ http://code.activestate.com/recipes/286222/ (r1) +import os + +_proc_status = '/proc/%d/status' % os.getpid() + +_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, + 'KB': 1024.0, 'MB': 1024.0*1024.0} + +def _VmB(VmKey): + '''Private. + ''' + global _proc_status, _scale + # get pseudo file /proc//status + try: + t = open(_proc_status) + v = t.read() + t.close() + except: + return 0.0 # non-Linux? + # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index(VmKey) + v = v[i:].split(None, 3) # whitespace + if len(v) < 3: + return 0.0 # invalid format? + # convert Vm value to bytes + return float(v[1]) * _scale[v[2]] + + +def memory(since=0.0): + '''Return memory usage in bytes. + ''' + return _VmB('VmSize:') - since + + +def resident(since=0.0): + '''Return resident memory usage in bytes. + ''' + return _VmB('VmRSS:') - since + + +def stacksize(since=0.0): + '''Return stack size in bytes. + ''' + return _VmB('VmStk:') - since +## end of http://code.activestate.com/recipes/286222/ }}} + + + From f71f60ab0cb84364a876dc7ee3950474e115d338 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 19:02:14 -0700 Subject: [PATCH 25/36] Edit metadata dialog: Fix memory leak caused by Next/Previous buttons --- src/calibre/gui2/dialogs/metadata_single.py | 47 +++++++-------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 9e3a8c7eda..4a9bb784c8 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -910,14 +910,18 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): bytes(self.splitter.saveState())) def break_cycles(self): - try: - self.view_format.disconnect() - except: - pass # Fails if view format was never connected - self.view_format = None - self.db = None - self.pi = None - self.cover_data = self.cpixmap = None + # Break any reference cycles that could prevent python + # from garbage collecting this dialog + def disconnect(signal): + try: + signal.disconnect() + except: + pass # Fails if view format was never connected + disconnect(self.view_format) + for b in ('next_button', 'prev_button'): + x = getattr(self, b, None) + if x is not None: + disconnect(x.clicked) if __name__ == '__main__': from calibre.library import db @@ -935,38 +939,17 @@ if __name__ == '__main__': d.reject() del d - for i in range(3): + for i in range(5): gc.collect() before = memory() - gc.collect() d = MetadataSingleDialog(None, 4, db) - d.break_cycles() d.reject() + d.break_cycles() del d - for i in range(3): + for i in range(5): gc.collect() print 'Used memory:', memory(before)/1024.**2, 'MB' - gc.collect() - ''' - nmap, omap = {}, {} - for x in objects: - omap[id(x)] = x - for x in nobjects: - nmap[id(x)] = x - new_ids = set(nmap.keys()) - set(omap.keys()) - print "New ids:", len(new_ids) - for i in new_ids: - o = nmap[i] - if o is objects: - continue - print repr(o)[:1050] - refs = gc.get_referrers(o) - for r in refs: - if r is objects or r is nobjects: - continue - print '\t', r - ''' From c11bbe3eea7c6003787b66b44e19a280686228ee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 22:06:37 -0700 Subject: [PATCH 26/36] Fix string.letters in non english locales --- src/calibre/library/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 8ff23c0a0a..177c5063ac 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -19,12 +19,15 @@ def generate_test_db(library_path, max_tags=10 ): import random, string, os, sys, time + from calibre.constants import preferred_encoding if not os.path.exists(library_path): os.makedirs(library_path) + letters = string.letters.decode(preferred_encoding) + def randstr(length): - return ''.join(random.choice(string.letters) for i in + return ''.join(random.choice(letters) for i in xrange(length)) all_tags = [randstr(tag_length) for j in xrange(num_of_tags)] From d6f5f634eacced09f7208613df58952f1bfe95e9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 13 Dec 2010 23:17:13 -0700 Subject: [PATCH 27/36] Allow changing the font used in the calibre interface via Preferences->Look and feel --- src/calibre/gui2/__init__.py | 7 ++- src/calibre/gui2/preferences/look_feel.py | 59 +++++++++++++++++++++-- src/calibre/gui2/preferences/look_feel.ui | 16 +++++- 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 57ca2a1880..f96c64080d 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -9,7 +9,7 @@ from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ QByteArray, QTranslator, QCoreApplication, QThread, \ QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \ - QIcon, QApplication, QDialog, QPushButton, QUrl + QIcon, QApplication, QDialog, QPushButton, QUrl, QFont ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' @@ -52,6 +52,7 @@ gprefs.defaults['show_splash_screen'] = True gprefs.defaults['toolbar_icon_size'] = 'medium' gprefs.defaults['toolbar_text'] = 'auto' gprefs.defaults['show_child_bar'] = False +gprefs.defaults['font'] = None # }}} @@ -613,6 +614,10 @@ class Application(QApplication): qt_app = self self._file_open_paths = [] self._file_open_lock = RLock() + self.original_font = QFont(QApplication.font()) + fi = gprefs['font'] + if fi is not None: + QApplication.setFont(QFont(*fi)) def _send_file_open_events(self): with self._file_open_lock: diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 10c2fcfe95..b2ba87d1e0 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -5,10 +5,11 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from PyQt4.Qt import QApplication, QFont, QFontInfo, QFontDialog from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.look_feel_ui import Ui_Form -from calibre.gui2 import config, gprefs +from calibre.gui2 import config, gprefs, qt_app from calibre.utils.localization import available_translations, \ get_language, get_lang from calibre.utils.config import prefs @@ -56,12 +57,64 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): (_('Never'), 'never')] r('toolbar_text', gprefs, choices=choices) + self.current_font = None + self.change_font_button.clicked.connect(self.change_font) + + + def initialize(self): + ConfigWidgetBase.initialize(self) + self.current_font = gprefs['font'] + self.update_font_display() + + def restore_defaults(self): + ConfigWidgetBase.restore_defaults(self) + ofont = self.current_font + self.current_font = None + if ofont is not None: + self.changed_signal.emit() + self.update_font_display() + + def build_font_obj(self): + font_info = self.current_font + if font_info is not None: + font = QFont(*font_info) + else: + font = qt_app.original_font + return font + + def update_font_display(self): + font = self.build_font_obj() + fi = QFontInfo(font) + name = unicode(fi.family()) + + self.font_display.setFont(font) + self.font_display.setText(_('Current font:') + ' ' + name + + ' [%dpt]'%fi.pointSize()) + + def change_font(self, *args): + fd = QFontDialog(self.build_font_obj(), self) + if fd.exec_() == fd.Accepted: + font = fd.selectedFont() + fi = QFontInfo(font) + self.current_font = (unicode(fi.family()), fi.pointSize(), + fi.weight(), fi.italic()) + self.update_font_display() + self.changed_signal.emit() + + def commit(self, *args): + rr = ConfigWidgetBase.commit(self, *args) + if self.current_font != gprefs['font']: + gprefs['font'] = self.current_font + QApplication.setFont(self.font_display.font()) + rr = True + return rr + + def refresh_gui(self, gui): gui.search.search_as_you_type(config['search_as_you_type']) - + self.update_font_display() if __name__ == '__main__': - from PyQt4.Qt import QApplication app = QApplication([]) test_widget('Interface', 'Look & Feel') diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index 1de55d51ef..91f45a155f 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -183,7 +183,7 @@ - + Qt::Vertical @@ -196,6 +196,20 @@ + + + + true + + + + + + + Change &font (needs restart) + + + From d2ff37b5a179ecf10041d8e51a810cb067820ca9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 14:11:10 +0000 Subject: [PATCH 28/36] Improved get_categories -- approximately 6 times faster --- src/calibre/library/database2.py | 219 ++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 32 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1229b60577..0d301ccaff 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -14,7 +14,7 @@ from operator import itemgetter from PyQt4.QtGui import QImage - +from calibre import prints from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase @@ -1039,43 +1039,170 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tn=field['table'], col=field['link_column']), (id_,)) return set(x[0] for x in ans) +########## data structures for get_categories + CATEGORY_SORTS = ('name', 'popularity', 'rating') - def get_categories(self, sort='name', ids=None, icon_map=None): - self.books_list_filter.change([] if not ids else ids) + class TCat_Tag(object): - categories = {} + def __init__(self, name, sort): + self.n = name + self.s = sort + self.c = 0 + self.rt = 0 + self.rc = 0 + self.id = None + + def set_all(self, c, rt, rc, id): + self.c = c + self.rt = rt + self.rc = rc + self.id = id + + def __str__(self): + return unicode(self) + + def __unicode__(self): + return 'n=%s s=%s c=%d rt=%d rc=%d id=%s'%\ + (self.n, self.s, self.c, self.rt, self.rc, self.id) + + + def get_categories(self, sort='name', ids=None, icon_map=None): + start = time.time() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') + if sort not in self.CATEGORY_SORTS: + raise ValueError('sort ' + sort + ' not a valid value') + + self.books_list_filter.change([] if not ids else ids) + id_filter = None if not ids else frozenset(ids) tb_cats = self.field_metadata - #### First, build the standard and custom-column categories #### + tcategories = {} + tids = {} + md = [] + + # First, build the maps. We need a category->items map and an + # item -> (item_id, sort_val) map to use in the books loop for category in tb_cats.keys(): cat = tb_cats[category] - if not cat['is_category'] or cat['kind'] in ['user', 'search']: + if not cat['is_category'] or cat['kind'] in ['user', 'search'] \ + or category in ['news', 'formats']: continue - tn = cat['table'] - categories[category] = [] #reserve the position in the ordered list - if tn is None: # Nothing to do for the moment + # Get the ids for the item values + if not cat['is_custom']: + funcs = { + 'authors' : self.get_authors_with_ids, + 'series' : self.get_series_with_ids, + 'publisher': self.get_publishers_with_ids, + 'tags' : self.get_tags_with_ids, + 'rating' : self.get_ratings_with_ids, + } + func = funcs.get(category, None) + if func: + list = func() + else: + raise ValueError(category + ' has no get with ids function') + else: + list = self.get_custom_items_with_ids(label=cat['label']) + tids[category] = {} + if category == 'authors': + for l in list: + (id, val, sort_val) = (l[0], l[1], l[2]) + tids[category][val] = (id, sort_val) + else: + for l in list: + (id, val) = (l[0], l[1]) + tids[category][val] = (id, val) + # add an empty category to the category map + tcategories[category] = {} + # create a list of category/field_index for the books scan to use. + # This saves iterating through field_metadata for each book + md.append((category, cat['rec_index'], cat['is_multiple'])) + + print 'end phase "collection":', time.time() - start, 'seconds' + + # Now scan every book looking for category items. + # Code below is duplicated because it shaves off 10% of the loop time + id_dex = self.FIELD_MAP['id'] + rating_dex = self.FIELD_MAP['rating'] + for book in self.data.iterall(): + if id_filter and book[id_dex] not in id_filter: continue - cn = cat['column'] - if ids is None: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_{1}'''.format(cn, tn) - else: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_filtered_{1}'''.format(cn, tn) - if sort == 'popularity': - query += ' ORDER BY count DESC, sort ASC' - elif sort == 'name': - query += ' ORDER BY sort COLLATE icucollate' - else: - query += ' ORDER BY avg_rating DESC, sort ASC' - data = self.conn.get(query) + rating = book[rating_dex] + # We kept track of all possible category field_map positions above + for (cat, dex, mult) in md: + if book[dex] is None: + continue + if not mult: + val = book[dex] + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + else: + vals = book[dex].split(mult) + for val in vals: + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + + print 'end phase "books":', time.time() - start, 'seconds' + + # Now do news + tcategories['news'] = {} + cat = tb_cats['news'] + tn = cat['table'] + cn = cat['column'] + if ids is None: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_{1}'''.format(cn, tn) + else: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_filtered_{1}'''.format(cn, tn) + # results will be sorted later + data = self.conn.get(query) + for r in data: + item = LibraryDatabase2.TCat_Tag(r[1], r[1]) + item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) + tcategories['news'][r[1]] = item + + print 'end phase "news":', time.time() - start, 'seconds' + + # Build the real category list by iterating over the temporary copy + # and building the Tag instances. + categories = {} + for category in tb_cats.keys(): + if category not in tcategories: + continue + cat = tb_cats[category] + + # prepare the place where we will put the array of Tags + categories[category] = [] # icon_map is not None if get_categories is to store an icon and # possibly a tooltip in the tag structure. - icon, tooltip = None, '' + icon = None + tooltip = '' label = tb_cats.key_to_label(category) if icon_map: if not tb_cats.is_custom_field(category): @@ -1087,23 +1214,40 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip = self.custom_column_label_map[label]['name'] datatype = cat['datatype'] - avgr = itemgetter(3) - item_not_zero_func = lambda x: x[2] > 0 + avgr = lambda x: 0.0 if x.rc == 0 else x.rt/x.rc + # Duplicate the build of items below to avoid using a lambda func + # in the main Tag loop. Saves a few % if datatype == 'rating': - # eliminate the zero ratings line as well as count == 0 - item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0) formatter = (lambda x:u'\u2605'*int(x/2)) - avgr = itemgetter(1) + avgr = lambda x : x.n + # eliminate the zero ratings line as well as count == 0 + items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0] elif category == 'authors': # Clean up the authors strings to human-readable form formatter = (lambda x: x.replace('|', ',')) + items = [v for v in tcategories[category].values() if v.c > 0] else: formatter = (lambda x:unicode(x)) + items = [v for v in tcategories[category].values() if v.c > 0] - categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0], - avg=avgr(r), sort=r[4], icon=icon, + # sort the list + if sort == 'name': + kf = lambda x: sort_key(x.s) if isinstance(x.s, unicode) else x.s + reverse=False + elif sort == 'popularity': + kf = lambda x: x.c + reverse=True + else: + kf = avgr + reverse=True + items.sort(key=kf, reverse=reverse) + + categories[category] = [Tag(formatter(r.n), count=r.c, id=r.id, + avg=avgr(r), sort=r.s, icon=icon, tooltip=tooltip, category=category) - for r in data if item_not_zero_func(r)] + for r in items] + + print 'end phase "tags list":', time.time() - start, 'seconds' # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1189,8 +1333,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items + t = time.time() - start + print 'get_categories ran in:', t, 'seconds' + return categories + ############# End get_categories + def tags_older_than(self, tag, delta): tag = tag.lower().strip() now = nowf() @@ -1486,6 +1635,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Note: we generally do not need to refresh_ids because library_view will # refresh everything. + def get_ratings_with_ids(self): + result = self.conn.get('SELECT id,rating FROM ratings') + if not result: + return [] + return result + def dirty_books_referencing(self, field, id, commit=True): # Get the list of books to dirty -- all books that reference the item table = self.field_metadata[field]['table'] From 2c652bdee745c36b504c761587b977b2d723624f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:00:45 -0700 Subject: [PATCH 29/36] Science based medicine by BuzzKill --- .../recipes/science_based_medicine.recipe | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 resources/recipes/science_based_medicine.recipe diff --git a/resources/recipes/science_based_medicine.recipe b/resources/recipes/science_based_medicine.recipe new file mode 100644 index 0000000000..7aa28cb170 --- /dev/null +++ b/resources/recipes/science_based_medicine.recipe @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag + +class SBM(BasicNewsRecipe): + title = 'Science Based Medicine' + __author__ = 'BuzzKill' + description = 'Exploring issues and controversies in the relationship between science and medicine' + oldest_article = 5 + max_articles_per_feed = 15 + no_stylesheets = True + use_embedded_content = False + encoding = 'utf-8' + publisher = 'SBM' + category = 'science, sbm, ebm, blog, pseudoscience' + language = 'en' + + lang = 'en-US' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : lang + , 'pretty_print' : True + } + + keep_only_tags = [ + dict(name='a', attrs={'title':re.compile(r'Posts by.*', re.DOTALL|re.IGNORECASE)}), + dict(name='div', attrs={'class':'entry'}) + ] + + feeds = [(u'Science Based Medicine', u'http://www.sciencebasedmedicine.org/?feed=rss2')] + + def preprocess_html(self, soup): + mtag = Tag(soup,'meta',[('http-equiv','Content-Type'),('context','text/html; charset=utf-8')]) + soup.head.insert(0,mtag) + soup.html['lang'] = self.lang + return self.adeify_images(soup) + From 032890d06f69a92040884059ea5093c901414406 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:18:03 -0700 Subject: [PATCH 30/36] Fix Zeit Online --- resources/recipes/zeitde.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/zeitde.recipe b/resources/recipes/zeitde.recipe index 64345ea675..c757c0d5bd 100644 --- a/resources/recipes/zeitde.recipe +++ b/resources/recipes/zeitde.recipe @@ -60,7 +60,7 @@ class ZeitDe(BasicNewsRecipe): for tag in soup.findAll(name=['ul','li']): tag.name = 'div' - soup.html['xml:lang'] = self.lang + soup.html['xml:lang'] = self.language.replace('_', '-') soup.html['lang'] = self.lang mtag = '' soup.head.insert(0,mtag) From 240b371ad5becc0d53bca8c9e3fae33af93a41a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:32:16 -0700 Subject: [PATCH 31/36] ... --- resources/recipes/zeitde.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/zeitde.recipe b/resources/recipes/zeitde.recipe index c757c0d5bd..389bdec670 100644 --- a/resources/recipes/zeitde.recipe +++ b/resources/recipes/zeitde.recipe @@ -61,7 +61,7 @@ class ZeitDe(BasicNewsRecipe): tag.name = 'div' soup.html['xml:lang'] = self.language.replace('_', '-') - soup.html['lang'] = self.lang + soup.html['lang'] = self.language.replace('_', '-') mtag = '' soup.head.insert(0,mtag) return soup From 65021a0f0e15efb85befb93a287220259a161eea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:35:48 -0700 Subject: [PATCH 32/36] Allow device drivers to show feedback to users if their open() methods fail --- src/calibre/devices/errors.py | 5 +++++ src/calibre/devices/interface.py | 3 +++ src/calibre/gui2/device.py | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/errors.py b/src/calibre/devices/errors.py index 7464d6635e..3d88eb741f 100644 --- a/src/calibre/devices/errors.py +++ b/src/calibre/devices/errors.py @@ -36,6 +36,11 @@ class UserFeedback(DeviceError): self.details = details self.msg = msg +class OpenFeedback(DeviceError): + def __init__(self, msg): + self.feedback_msg = msg + DeviceError.__init__(self, msg) + class DeviceBusy(ProtocolError): """ Raised when device is busy """ def __init__(self, uerr=""): diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 48d751fc29..2a92f46e8d 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -216,6 +216,9 @@ class DevicePlugin(Plugin): an implementation of this function that should serve as a good example for USB Mass storage devices. + + This method can raise an OpenFeedback exception to display a message to + the user. ''' raise NotImplementedError() diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 008649f534..9d66f8fc0d 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -12,7 +12,7 @@ from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \ from calibre.customize.ui import available_input_formats, available_output_formats, \ device_plugins from calibre.devices.interface import DevicePlugin -from calibre.devices.errors import UserFeedback +from calibre.devices.errors import UserFeedback, OpenFeedback from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.utils.ipc.job import BaseJob from calibre.devices.scanner import DeviceScanner @@ -163,6 +163,9 @@ class DeviceManager(Thread): # {{{ dev.reset(detected_device=detected_device, report_progress=self.report_progress) dev.open() + except OpenFeedback, e: + self.open_feedback_slot(e.feedback_msg) + continue except: tb = traceback.format_exc() if DEBUG or tb not in self.reported_errors: From 0bd30a28d26e54c0b48e5ba3c311509744bbbd56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:43:33 -0700 Subject: [PATCH 33/36] Show open feedback message in a separate dialog --- src/calibre/gui2/device.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 9d66f8fc0d..07bfeccc4f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -122,7 +122,8 @@ def device_name_for_plugboards(device_class): class DeviceManager(Thread): # {{{ - def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2): + def __init__(self, connected_slot, job_manager, open_feedback_slot, + open_feedback_msg, sleep_time=2): ''' :sleep_time: Time to sleep between device probes in secs ''' @@ -143,6 +144,7 @@ class DeviceManager(Thread): # {{{ self.ejected_devices = set([]) self.mount_connection_requests = Queue.Queue(0) self.open_feedback_slot = open_feedback_slot + self.open_feedback_msg = open_feedback_msg def report_progress(self, *args): pass @@ -164,7 +166,7 @@ class DeviceManager(Thread): # {{{ report_progress=self.report_progress) dev.open() except OpenFeedback, e: - self.open_feedback_slot(e.feedback_msg) + self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg) continue except: tb = traceback.format_exc() @@ -597,11 +599,16 @@ class DeviceMixin(object): # {{{ _('Error communicating with device'), ' ') self.device_error_dialog.setModal(Qt.NonModal) self.device_manager = DeviceManager(Dispatcher(self.device_detected), - self.job_manager, Dispatcher(self.status_bar.show_message)) + self.job_manager, Dispatcher(self.status_bar.show_message), + Dispatcher(self.show_open_feedback)) self.device_manager.start() if tweaks['auto_connect_to_folder']: self.connect_to_folder_named(tweaks['auto_connect_to_folder']) + def show_open_feedback(self, devname, msg): + self.__of_dev_mem__ = d = info_dialog(self, devname, msg) + d.show() + def auto_convert_question(self, msg, autos): autos = u'\n'.join(map(unicode, map(force_unicode, autos))) return self.ask_a_yes_no_question( From 8f2171033a814dc7a47daf7388fc95ed5d7f382e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 18:05:54 +0000 Subject: [PATCH 34/36] Eliminate 2 superfluous calls to recount on startup -- 1 when not using a startup restriction and another immediately after initializing the model. --- src/calibre/gui2/tag_view.py | 1 + src/calibre/gui2/ui.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index d6c0156f13..f75061da12 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -101,6 +101,7 @@ class TagsView(QTreeView): # {{{ hidden_categories=self.hidden_categories, search_restriction=None, drag_drop_finished=self.drag_drop_finished) + self.pane_is_visible = True # because TagsModel.init did a recount self.sort_by = sort_by self.tag_match = tag_match self.db = db diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index cb25f75d4a..7279b7f8df 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -234,7 +234,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ######################### Search Restriction ########################## SearchRestrictionMixin.__init__(self) - self.apply_named_search_restriction(db.prefs['gui_restriction']) + if db.prefs['gui_restriction']: + self.apply_named_search_restriction(db.prefs['gui_restriction']) ########################### Cover Flow ################################ From b7e9749610d47b3f762943e5d613bb09777971db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:10:13 -0700 Subject: [PATCH 35/36] Use time.clock rather than time.time for timing --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 64d143ef3c..098cb04727 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.time() + start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "collection":', time.clock() - last, 'seconds' + last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "books":', time.clock() - last, 'seconds' + last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "news":', time.clock() - last, 'seconds' + last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "tags list":', time.clock() - last, 'seconds' + last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.time() - last, 'seconds' - print 'get_categories ran in:', time.time() - start, 'seconds' + print 'last phase ran in:', time.clock() - last, 'seconds' + print 'get_categories ran in:', time.clock() - start, 'seconds' return categories From 88264f6650b09b345870b64dcb3b1c8d4ea00dd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:26:41 -0700 Subject: [PATCH 36/36] Remove timing code from get_categories --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 098cb04727..33e4295f05 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.clock() + #start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "collection":', time.clock() - last, 'seconds' + #last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "books":', time.clock() - last, 'seconds' + #last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "news":', time.clock() - last, 'seconds' + #last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "tags list":', time.clock() - last, 'seconds' + #last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.clock() - last, 'seconds' - print 'get_categories ran in:', time.clock() - start, 'seconds' + #print 'last phase ran in:', time.clock() - last, 'seconds' + #print 'get_categories ran in:', time.clock() - start, 'seconds' return categories