From ff6c024d2bf2947906b82241415ead9e9caeced8 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Thu, 9 Dec 2010 23:48:57 +0900 Subject: [PATCH 01/15] add Kahoku Shinpo News and pet cat blog --- resources/recipes/kahokushinpo.recipe | 32 ++++++++++++++++++++++++ resources/recipes/uninohimitu.recipe | 36 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 resources/recipes/kahokushinpo.recipe create mode 100644 resources/recipes/uninohimitu.recipe diff --git a/resources/recipes/kahokushinpo.recipe b/resources/recipes/kahokushinpo.recipe new file mode 100644 index 0000000000..6e084d83cc --- /dev/null +++ b/resources/recipes/kahokushinpo.recipe @@ -0,0 +1,32 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +www.kahoku.co.jp +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + + +class KahokuShinpoNews(BasicNewsRecipe): + title = u'Kahoku Shinpo News' + __author__ = 'Hiroshi Miura' + oldest_article = 2 + max_articles_per_feed = 20 + description = 'Tohoku regional news paper in Japan' + publisher = 'Kahoku Shinpo Sha' + category = 'news, japan' + language = 'ja' + encoding = 'Shift_JIS' + + + feeds = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')] + + keep_only_tags = [ dict(id="page_title"), + dict(id="news_detail"), + dict(id="bt_title"), + {'class':"photoLeft"}, + dict(id="bt_body") + ] + remove_tags = [ {'class':"button"}] + diff --git a/resources/recipes/uninohimitu.recipe b/resources/recipes/uninohimitu.recipe new file mode 100644 index 0000000000..aac412744c --- /dev/null +++ b/resources/recipes/uninohimitu.recipe @@ -0,0 +1,36 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +http://ameblo.jp/sauta19/ +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class UniNoHimituKichiBlog(BasicNewsRecipe): + title = u'Uni secret base' + __author__ = 'Hiroshi Miura' + oldest_article = 2 + publication_type = 'blog' + max_articles_per_feed = 20 + description = 'Japanese famous Cat blog' + publisher = '' + category = 'cat, pet, japan' + language = 'ja' + encoding = 'utf-8' + + feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/sauta19/rss20.xml')] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'rssad.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds + From 34df6efff9256813718a12174ada30e04311867b Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Fri, 10 Dec 2010 09:50:09 +0900 Subject: [PATCH 02/15] recipe: add popular blog about internet technologies. --- resources/recipes/ajiajin.recipe | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 resources/recipes/ajiajin.recipe diff --git a/resources/recipes/ajiajin.recipe b/resources/recipes/ajiajin.recipe new file mode 100644 index 0000000000..c5f052982b --- /dev/null +++ b/resources/recipes/ajiajin.recipe @@ -0,0 +1,24 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +ajiajin.com/blog +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class AjiajinBlog(BasicNewsRecipe): + title = u'Ajiajin blog' + __author__ = 'Hiroshi Miura' + oldest_article = 5 + publication_type = 'blog' + max_articles_per_feed = 100 + description = 'The next generation internet trends in Japan and Asia' + publisher = '' + category = 'internet, asia, japan' + language = 'en' + encoding = 'utf-8' + + feeds = [(u'blog', u'http://feeds.feedburner.com/Asiajin')] + + From ee5e7abe0b77b6566cf1f215fcac4fe5b49ed697 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sat, 11 Dec 2010 11:30:22 +0900 Subject: [PATCH 03/15] recipe: Nikkei social - fix typo in title and function name --- resources/recipes/nikkei_sub_shakai.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/recipes/nikkei_sub_shakai.recipe b/resources/recipes/nikkei_sub_shakai.recipe index ed86493265..9a53e910e6 100644 --- a/resources/recipes/nikkei_sub_shakai.recipe +++ b/resources/recipes/nikkei_sub_shakai.recipe @@ -10,8 +10,8 @@ import mechanize from calibre.ptempfile import PersistentTemporaryFile -class NikkeiNet_sub_life(BasicNewsRecipe): - title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)' +class NikkeiNet_sub_shakai(BasicNewsRecipe): + title = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(Social)' __author__ = 'Hiroshi Miura' description = 'News and current market affairs from Japan' cover_url = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg' From a43274e55a4060bf864ecf1c8f54c64b0c3cee5f Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 12:56:52 +0900 Subject: [PATCH 04/15] recipe: add paper.li recipes --- resources/recipes/paperli.recipe | 58 +++++++++++++++++++++++++ resources/recipes/paperli_topic.recipe | 59 ++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 resources/recipes/paperli.recipe create mode 100644 resources/recipes/paperli_topic.recipe diff --git a/resources/recipes/paperli.recipe b/resources/recipes/paperli.recipe new file mode 100644 index 0000000000..2c99e5dc81 --- /dev/null +++ b/resources/recipes/paperli.recipe @@ -0,0 +1,58 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +paperli +''' + +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +import re, sys + +class paperli(BasicNewsRecipe): +#-------------------please change here ---------------- + paperli_tag = 'osm' + title = u'The # osm Daily - paperli' +#------------------------------------------------------------- + base_url = 'http://paper.li' + index = '/tag/'+paperli_tag+'/~list' + + __author__ = 'Hiroshi Miura' + oldest_article = 7 + max_articles_per_feed = 100 + description = 'paper.li page' + publisher = 'paper.li' + category = 'paper.li' + language = 'en' + encoding = 'utf-8' + remove_javascript = True + timefmt = '[%y/%m/%d]' + + def parse_index(self): + feeds = [] + newsarticles = [] + topic = 'HEADLINE' + + #for pages + page = self.index + while True: + soup = self.index_to_soup(''.join([self.base_url,page])) + for itt in soup.findAll('div',attrs={'class':'yui-u'}): + itema = itt.find('a',href=True,attrs={'class':'ts'}) + if itema is not None: + itemd = itt.find('div',text=True, attrs={'class':'text'}) + newsarticles.append({ + 'title' :itema.string + ,'date' :strftime(self.timefmt) + ,'url' :itema['href'] + ,'description':itemd.string + }) + + nextpage = soup.find('div',attrs={'class':'pagination_top'}).find('li', attrs={'class':'next'}) + if nextpage is not None: + page = nextpage.find('a', href=True)['href'] + else: + break + + feeds.append((topic, newsarticles)) + return feeds + diff --git a/resources/recipes/paperli_topic.recipe b/resources/recipes/paperli_topic.recipe new file mode 100644 index 0000000000..3906af362f --- /dev/null +++ b/resources/recipes/paperli_topic.recipe @@ -0,0 +1,59 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +paperli +''' + +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +import re + +class paperli_topics(BasicNewsRecipe): +#-------------------please change here ---------------- + paperli_tag = 'wikileaks' + title = u'The # wikileaks Daily - paperli' +#------------------------------------------------------------- + __author__ = 'Hiroshi Miura' + oldest_article = 7 + max_articles_per_feed = 100 + description = 'paper.li page about '+ paperli_tag + publisher = 'paper.li' + category = 'paper.li' + language = 'en' + encoding = 'utf-8' + remove_javascript = True + masthead_title = u'The '+ paperli_tag +' Daily' + timefmt = '[%y/%m/%d]' + base_url = 'http://paper.li' + index = base_url+'/tag/'+paperli_tag + + + def parse_index(self): + + # get topics + topics = [] + soup = self.index_to_soup(self.index) + topics_lists = soup.find('div',attrs={'class':'paper-nav-bottom'}) + for item in topics_lists.findAll('li', attrs={'class':""}): + itema = item.find('a',href=True) + topics.append({'title': itema.string, 'url': itema['href']}) + + #get feeds + feeds = [] + for topic in topics: + newsarticles = [] + soup = self.index_to_soup(''.join([self.base_url, topic['url'] ])) + topstories = soup.findAll('div',attrs={'class':'yui-u'}) + for itt in topstories: + itema = itt.find('a',href=True,attrs={'class':'ts'}) + if itema is not None: + itemd = itt.find('div',text=True, attrs={'class':'text'}) + newsarticles.append({ + 'title' :itema.string + ,'date' :strftime(self.timefmt) + ,'url' :itema['href'] + ,'description':itemd.string + }) + feeds.append((topic['title'], newsarticles)) + return feeds + From 1efd975625c1f32e52722f7ab18e3f099496c274 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 12:58:32 +0900 Subject: [PATCH 05/15] recipe: fix kahoku shinpo --- resources/recipes/kahokushinpo.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/recipes/kahokushinpo.recipe b/resources/recipes/kahokushinpo.recipe index 6e084d83cc..172014d3a0 100644 --- a/resources/recipes/kahokushinpo.recipe +++ b/resources/recipes/kahokushinpo.recipe @@ -9,7 +9,7 @@ from calibre.web.feeds.news import BasicNewsRecipe class KahokuShinpoNews(BasicNewsRecipe): - title = u'Kahoku Shinpo News' + title = u'\u6cb3\u5317\u65b0\u5831' __author__ = 'Hiroshi Miura' oldest_article = 2 max_articles_per_feed = 20 @@ -18,7 +18,7 @@ class KahokuShinpoNews(BasicNewsRecipe): category = 'news, japan' language = 'ja' encoding = 'Shift_JIS' - + no_stylesheets = True feeds = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')] From d18bef33e11c20be510339e9ffe7bca665ff6dde Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 22:28:55 +0900 Subject: [PATCH 06/15] recipe: add national geographic news - national geographic Japan - national geographic News --- resources/recipes/nationalgeographic.recipe | 38 +++++++++++++++++++ resources/recipes/nationalgeographicjp.recipe | 20 ++++++++++ 2 files changed, 58 insertions(+) create mode 100644 resources/recipes/nationalgeographic.recipe create mode 100644 resources/recipes/nationalgeographicjp.recipe diff --git a/resources/recipes/nationalgeographic.recipe b/resources/recipes/nationalgeographic.recipe new file mode 100644 index 0000000000..b540f9b044 --- /dev/null +++ b/resources/recipes/nationalgeographic.recipe @@ -0,0 +1,38 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +nationalgeographic.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class NationalGeographicNews(BasicNewsRecipe): + title = u'National Geographic News' + oldest_article = 7 + max_articles_per_feed = 100 + remove_javascript = True + no_stylesheets = True + use_embedded_content = False + + feeds = [(u'news', u'http://feeds.nationalgeographic.com/ng/News/News_Main')] + + remove_tags_before = dict(id='page_head') + remove_tags_after = [dict(id='social_buttons'),{'class':'aside'}] + remove_tags = [ + {'class':'hidden'} + + ] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'ads\.pheedo\.com', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds diff --git a/resources/recipes/nationalgeographicjp.recipe b/resources/recipes/nationalgeographicjp.recipe new file mode 100644 index 0000000000..5798acb102 --- /dev/null +++ b/resources/recipes/nationalgeographicjp.recipe @@ -0,0 +1,20 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +nationalgeographic.co.jp +''' + +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class NationalGeoJp(BasicNewsRecipe): + title = u'\u30ca\u30b7\u30e7\u30ca\u30eb\u30fb\u30b8\u30aa\u30b0\u30e9\u30d5\u30a3\u30c3\u30af\u30cb\u30e5\u30fc\u30b9' + oldest_article = 7 + max_articles_per_feed = 100 + no_stylesheets = True + + feeds = [(u'news', u'http://www.nationalgeographic.co.jp/news/rss.php')] + + def print_version(self, url): + return re.sub(r'news_article.php','news_printer_friendly.php', url) + From c3bbe2cc8659db1c13bf4f001c09bb3e3f658145 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Sun, 12 Dec 2010 22:46:55 +0900 Subject: [PATCH 07/15] recipe: add dog blog in Japanese --- resources/recipes/chouchoublog.recipe | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 resources/recipes/chouchoublog.recipe diff --git a/resources/recipes/chouchoublog.recipe b/resources/recipes/chouchoublog.recipe new file mode 100644 index 0000000000..8c953deef0 --- /dev/null +++ b/resources/recipes/chouchoublog.recipe @@ -0,0 +1,37 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Hiroshi Miura ' +''' +http://ameblo.jp/ +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class SakuraBlog(BasicNewsRecipe): + title = u'chou chou blog' + __author__ = 'Hiroshi Miura' + oldest_article = 4 + publication_type = 'blog' + max_articles_per_feed = 20 + description = 'Japanese popular dog blog' + publisher = '' + category = 'dog, pet, japan' + language = 'ja' + encoding = 'utf-8' + use_embedded_content = True + + feeds = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/chouchou1218/rss20.xml')] + + def parse_feeds(self): + feeds = BasicNewsRecipe.parse_feeds(self) + for curfeed in feeds: + delList = [] + for a,curarticle in enumerate(curfeed.articles): + if re.search(r'rssad.jp', curarticle.url): + delList.append(curarticle) + if len(delList)>0: + for d in delList: + index = curfeed.articles.index(d) + curfeed.articles[index:index+1] = [] + return feeds + From 9ea944ff143315d88dced7a5aae538566cb1a730 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:35:12 -0700 Subject: [PATCH 08/15] Conversion pipeline: Add an option to set the minimum line height of all elemnts as a percentage of the computed font size. By default, calibre now sets the line height to 120% of the computed font size. --- src/calibre/ebooks/conversion/cli.py | 2 +- src/calibre/ebooks/conversion/plumber.py | 25 ++++++++-- src/calibre/ebooks/oeb/stylizer.py | 6 +-- src/calibre/ebooks/oeb/transforms/flatcss.py | 11 +++++ src/calibre/gui2/convert/look_and_feel.py | 2 +- src/calibre/gui2/convert/look_and_feel.ui | 51 ++++++++++++++------ 6 files changed, 74 insertions(+), 23 deletions(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index 62a941142b..3178fe1b43 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -120,7 +120,7 @@ def add_pipeline_options(parser, plumber): [ 'base_font_size', 'disable_font_rescaling', 'font_size_mapping', - 'line_height', + 'line_height', 'minimum_line_height', 'linearize_tables', 'extra_css', 'smarten_punctuation', 'margin_top', 'margin_left', 'margin_right', diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 9a863d7e66..f5beba375d 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -160,13 +160,30 @@ OptionRecommendation(name='disable_font_rescaling', ) ), +OptionRecommendation(name='minimum_line_height', + recommended_value=120.0, level=OptionRecommendation.LOW, + help=_( + 'The minimum line height, as a percentage of the element\'s ' + 'calculated font size. calibre will ensure that every element ' + 'has a line height of at least this setting, irrespective of ' + 'what the input document specifies. Set to zero to disable. ' + 'Default is 120%. Use this setting in preference to ' + 'the direct line height specification, unless you know what ' + 'you are doing. For example, you can achieve "double spaced" ' + 'text by setting this to 240.' + ) + ), + OptionRecommendation(name='line_height', recommended_value=0, level=OptionRecommendation.LOW, - help=_('The line height in pts. Controls spacing between consecutive ' - 'lines of text. By default no line height manipulation is ' - 'performed.' - ) + help=_( + 'The line height in pts. Controls spacing between consecutive ' + 'lines of text. Only applies to elements that do not define ' + 'their own line height. In most cases, the minimum line height ' + 'option is more useful. ' + 'By default no line height manipulation is performed.' + ) ), OptionRecommendation(name='linearize_tables', diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 6c0c384eb3..616cd3b800 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -633,12 +633,12 @@ class Style(object): parent = self._getparent() if 'line-height' in self._style: lineh = self._style['line-height'] + if lineh == 'normal': + lineh = '1.2' try: - float(lineh) + result = float(lineh) * self.fontSize except ValueError: result = self._unit_convert(lineh, base=self.fontSize) - else: - result = float(lineh) * self.fontSize elif parent is not None: # TODO: proper inheritance result = parent.lineHeight diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 7b83421097..653aa4533b 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -245,6 +245,8 @@ class CSSFlattener(object): del node.attrib['bgcolor'] if cssdict.get('font-weight', '').lower() == 'medium': cssdict['font-weight'] = 'normal' # ADE chokes on font-weight medium + + fsize = font_size if not self.context.disable_font_rescaling: _sbase = self.sbase if self.sbase is not None else \ self.context.source.fbase @@ -258,6 +260,14 @@ class CSSFlattener(object): fsize = self.fmap[font_size] cssdict['font-size'] = "%0.5fem" % (fsize / psize) psize = fsize + + try: + minlh = self.context.minimum_line_height / 100. + if style['line-height'] < minlh * fsize: + cssdict['line-height'] = str(minlh) + except: + self.oeb.logger.exception('Failed to set minimum line-height') + if cssdict: if self.lineh and self.fbase and tag != 'body': self.clean_edges(cssdict, style, psize) @@ -290,6 +300,7 @@ class CSSFlattener(object): lineh = self.lineh / psize cssdict['line-height'] = "%0.5fem" % lineh + if (self.context.remove_paragraph_spacing or self.context.insert_blank_line) and tag in ('p', 'div'): if item_id != 'calibre_jacket' or self.context.output_profile.name == 'Kindle': diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index ec3f0b944d..98b9cb8155 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -21,7 +21,7 @@ class LookAndFeelWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, ['change_justification', 'extra_css', 'base_font_size', - 'font_size_mapping', 'line_height', + 'font_size_mapping', 'line_height', 'minimum_line_height', 'linearize_tables', 'smarten_punctuation', 'disable_font_rescaling', 'insert_blank_line', 'remove_paragraph_spacing', 'remove_paragraph_spacing_indent_size','input_encoding', diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index c683300854..367233e2c0 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -97,7 +97,7 @@ - + Line &height: @@ -107,7 +107,7 @@ - + pt @@ -117,7 +117,7 @@ - + Input character &encoding: @@ -127,17 +127,17 @@ - + - + Remove &spacing between paragraphs - + @@ -164,21 +164,21 @@ - + Text justification: - + &Linearize tables - + Extra &CSS @@ -190,37 +190,60 @@ - + - + &Transliterate unicode characters to ASCII - + Insert &blank line - + Keep &ligatures - + Smarten &punctuation + + + + Minimum &line height: + + + opt_minimum_line_height + + + + + + + % + + + 1 + + + 900.000000000000000 + + + From e177f043c0527d88807fe01ca2d31329bddff660 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:36:47 -0700 Subject: [PATCH 09/15] TXT Input: Use a nicer name for the generated html file --- src/calibre/ebooks/txt/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index b444bf1cf4..e82b980009 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,7 +77,7 @@ class TXTInput(InputFormatPlugin): base = os.getcwdu() if hasattr(stream, 'name'): base = os.path.dirname(stream.name) - htmlfile = open(os.path.join(base, 'temp_calibre_txt_input_to_html.html'), + htmlfile = open(os.path.join(base, 'index.html'), 'wb') htmlfile.write(html.encode('utf-8')) htmlfile.close() From 2dd9b3f128bfa7f4f5cbe20744c0426af4895284 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 10:45:55 -0700 Subject: [PATCH 10/15] ... --- src/calibre/ebooks/txt/input.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/txt/input.py index e82b980009..44b98304ea 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/txt/input.py @@ -77,10 +77,14 @@ class TXTInput(InputFormatPlugin): base = os.getcwdu() if hasattr(stream, 'name'): base = os.path.dirname(stream.name) - htmlfile = open(os.path.join(base, 'index.html'), - 'wb') - htmlfile.write(html.encode('utf-8')) - htmlfile.close() + fname = os.path.join(base, 'index.html') + c = 0 + while os.path.exists(fname): + c += 1 + fname = 'index%d.html'%c + htmlfile = open(fname, 'wb') + with htmlfile: + htmlfile.write(html.encode('utf-8')) cwd = os.getcwdu() odi = options.debug_pipeline options.debug_pipeline = None From 4a8551962fdbe65ba4c77a7e08f0ae1baa4f948b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 11:07:31 -0700 Subject: [PATCH 11/15] Edit metadata dialog: clean up handling of cover fetch thread --- src/calibre/gui2/dialogs/metadata_single.py | 43 ++++++++++----------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index 3205b1d23c..d9bb1c2a33 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -240,37 +240,39 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.cover_fetcher = CoverFetcher(None, None, isbn, self.timeout, title, author) self.cover_fetcher.start() - self._hangcheck = QTimer(self) - self._hangcheck.timeout.connect(self.hangcheck, - type=Qt.QueuedConnection) self.cf_start_time = time.time() self.pi.start(_('Downloading cover...')) - self._hangcheck.start(100) + QTimer.singleShot(100, self.hangcheck) def hangcheck(self): - if self.cover_fetcher.is_alive() and \ - time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: + cf = self.cover_fetcher + if cf is None: + # Called after dialog closed + return + + if cf.is_alive() and \ + time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: + QTimer.singleShot(100, self.hangcheck) return - self._hangcheck.stop() try: - if self.cover_fetcher.is_alive(): + if cf.is_alive(): error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
')+ _('The download timed out.')).exec_() return - if self.cover_fetcher.needs_isbn: + if cf.needs_isbn: error_dialog(self, _('Cannot fetch cover'), _('Could not find cover for this book. Try ' 'specifying the ISBN first.')).exec_() return - if self.cover_fetcher.exception is not None: - err = self.cover_fetcher.exception + if cf.exception is not None: + err = cf.exception error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
')+unicode(err)).exec_() return - if self.cover_fetcher.errors and self.cover_fetcher.cover_data is None: - details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in self.cover_fetcher.errors]) + if cf.errors and cf.cover_data is None: + details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in cf.errors]) error_dialog(self, _('Cannot fetch cover'), _('Could not fetch cover.
') + _('For the error message from each cover source, ' @@ -278,7 +280,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): return pix = QPixmap() - pix.loadFromData(self.cover_fetcher.cover_data) + pix.loadFromData(cf.cover_data) if pix.isNull(): error_dialog(self, _('Bad cover'), _('The cover is not a valid picture')).exec_() @@ -287,7 +289,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.update_cover_tooltip() self.cover_changed = True self.cpixmap = pix - self.cover_data = self.cover_fetcher.cover_data + self.cover_data = cf.cover_data finally: self.fetch_cover_button.setEnabled(True) self.unsetCursor() @@ -438,6 +440,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): def __init__(self, window, row, db, prev=None, next_=None): ResizableDialog.__init__(self, window) + self.cover_fetcher = None self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) self.cancel_all = False base = unicode(self.author_sort.toolTip()) @@ -828,10 +831,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.accept() def accept(self): - cf = getattr(self, 'cover_fetcher', None) - if cf is not None and hasattr(cf, 'terminate'): - cf.terminate() - cf.wait() try: if self.formats_changed: self.sync_formats() @@ -888,14 +887,12 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): show=True) raise self.save_state() + self.cover_fetcher = None QDialog.accept(self) def reject(self, *args): - cf = getattr(self, 'cover_fetcher', None) - if cf is not None and hasattr(cf, 'terminate'): - cf.terminate() - cf.wait() self.save_state() + self.cover_fetcher = None QDialog.reject(self, *args) def read_state(self): From 0e70bb8b592afbb950b05af6a99adc2afe5682d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 11:45:19 -0700 Subject: [PATCH 12/15] Preferences: Add tooltips to buddy labels as well. Fixes #7873 (No tooltip for some format conversion fields) --- src/calibre/gui2/convert/__init__.py | 17 ++++++++++++++++- src/calibre/gui2/preferences/main.py | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index 0c3a8b5a4e..c1efe5b9af 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -10,7 +10,7 @@ import textwrap from functools import partial from PyQt4.Qt import QWidget, QSpinBox, QDoubleSpinBox, QLineEdit, QTextEdit, \ - QCheckBox, QComboBox, Qt, QIcon, pyqtSignal + QCheckBox, QComboBox, Qt, QIcon, pyqtSignal, QLabel from calibre.customize.conversion import OptionRecommendation from calibre.ebooks.conversion.config import load_defaults, \ @@ -81,6 +81,21 @@ class Widget(QWidget): self.apply_recommendations(defaults) self.setup_help(get_help) + def process_child(child): + for g in child.children(): + if isinstance(g, QLabel): + buddy = g.buddy() + if buddy is not None and hasattr(buddy, '_help'): + g._help = buddy._help + htext = unicode(buddy.toolTip()).strip() + g.setToolTip(htext) + g.setWhatsThis(htext) + g.__class__.enterEvent = lambda obj, event: self.set_help(getattr(obj, '_help', obj.toolTip())) + else: + process_child(g) + process_child(self) + + def restore_defaults(self, get_option): defaults = GuiRecommendations() defaults.merge_recommendations(get_option, OptionRecommendation.LOW, diff --git a/src/calibre/gui2/preferences/main.py b/src/calibre/gui2/preferences/main.py index fc01a33cf6..f7d49427c8 100644 --- a/src/calibre/gui2/preferences/main.py +++ b/src/calibre/gui2/preferences/main.py @@ -251,10 +251,28 @@ class Preferences(QMainWindow): self.close() self.run_wizard_requested.emit() + def set_tooltips_for_labels(self): + + def process_child(child): + for g in child.children(): + if isinstance(g, QLabel): + buddy = g.buddy() + if buddy is not None and hasattr(buddy, 'toolTip'): + htext = unicode(buddy.toolTip()).strip() + etext = unicode(g.toolTip()).strip() + if htext and not etext: + g.setToolTip(htext) + g.setWhatsThis(htext) + else: + process_child(g) + + process_child(self.showing_widget) + def show_plugin(self, plugin): self.showing_widget = plugin.create_widget(self.scroll_area) self.showing_widget.genesis(self.gui) self.showing_widget.initialize() + self.set_tooltips_for_labels() self.scroll_area.setWidget(self.showing_widget) self.stack.setCurrentIndex(1) self.showing_widget.show() From 524e9f4def3e41d8f00644262db1c4cffdab3984 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:09:43 -0700 Subject: [PATCH 13/15] Implement #7877 (margin option in epub to mobi conversions) --- src/calibre/ebooks/mobi/mobiml.py | 2 +- src/calibre/ebooks/mobi/output.py | 6 ++++++ src/calibre/gui2/convert/mobi_output.py | 1 + src/calibre/gui2/convert/mobi_output.ui | 11 +++++++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 8d20179250..001cf2c1e9 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -184,7 +184,7 @@ class MobiMLizer(object): para.attrib['value'] = str(istates[-2].list_num) elif tag in NESTABLE_TAGS and istate.rendered: para = wrapper = bstate.nested[-1] - elif left > 0 and indent >= 0: + elif not self.opts.mobi_ignore_margins and left > 0 and indent >= 0: ems = self.profile.mobi_ems_per_blockquote para = wrapper = etree.SubElement(parent, XHTML('blockquote')) para = wrapper diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 4159c6dd40..a6f6c52b7f 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -39,6 +39,12 @@ class MOBIOutput(OutputFormatPlugin): OptionRecommendation(name='personal_doc', recommended_value='[PDOC]', help=_('Tag marking book to be filed with Personal Docs') ), + OptionRecommendation(name='mobi_ignore_margins', + recommended_value=False, + help=_('Ignore margins in the input document. If False, then ' + 'the MOBI output plugin will try to convert margins specified' + ' in the input document, otherwise it will ignore them.') + ), ]) def check_for_periodical(self): diff --git a/src/calibre/gui2/convert/mobi_output.py b/src/calibre/gui2/convert/mobi_output.py index 23c0b30253..14aca24db5 100644 --- a/src/calibre/gui2/convert/mobi_output.py +++ b/src/calibre/gui2/convert/mobi_output.py @@ -25,6 +25,7 @@ class PluginWidget(Widget, Ui_Form): def __init__(self, parent, get_option, get_help, db=None, book_id=None): Widget.__init__(self, parent, ['prefer_author_sort', 'rescale_images', 'toc_title', + 'mobi_ignore_margins', 'dont_compress', 'no_inline_toc', 'masthead_font','personal_doc'] ) self.db, self.book_id = db, book_id diff --git a/src/calibre/gui2/convert/mobi_output.ui b/src/calibre/gui2/convert/mobi_output.ui index 176ce681c0..e9eab45e1a 100644 --- a/src/calibre/gui2/convert/mobi_output.ui +++ b/src/calibre/gui2/convert/mobi_output.ui @@ -55,7 +55,7 @@ - + Kindle options @@ -101,7 +101,7 @@ - + Qt::Vertical @@ -114,6 +114,13 @@ + + + + Ignore &margins + + + From e56e2c7f45580352d71ab93a8337fdd3678ee9fc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:24:26 -0700 Subject: [PATCH 14/15] Improved recipe for Dilbert --- resources/recipes/dilbert.recipe | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/resources/recipes/dilbert.recipe b/resources/recipes/dilbert.recipe index 82966b1d15..2c3268da2f 100644 --- a/resources/recipes/dilbert.recipe +++ b/resources/recipes/dilbert.recipe @@ -3,15 +3,16 @@ __copyright__ = '2009, Darko Miletic ' ''' http://www.dilbert.com ''' -import re from calibre.web.feeds.recipes import BasicNewsRecipe +import re -class DosisDiarias(BasicNewsRecipe): +class DilbertBig(BasicNewsRecipe): title = 'Dilbert' - __author__ = 'Darko Miletic' + __author__ = 'Darko Miletic and Starson17' description = 'Dilbert' - oldest_article = 5 + reverse_article_order = True + oldest_article = 15 max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = True @@ -29,20 +30,23 @@ class DosisDiarias(BasicNewsRecipe): feeds = [(u'Dilbert', u'http://feeds.dilbert.com/DilbertDailyStrip' )] - preprocess_regexps = [ - (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), - lambda match: 'strip.zoom.gif') - ] - - def get_article_url(self, article): return article.get('feedburner_origlink', None) + preprocess_regexps = [ + (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), lambda match: 'strip.zoom.gif') + ] + def preprocess_html(self, soup): for tag in soup.findAll(name='a'): if tag['href'].find('http://feedads') >= 0: tag.extract() return soup - - + extra_css = ''' + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + img {max-width:100%; min-width:100%;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} + ''' From 86ee4489d4a99c99930c646dbcc273b920b7a821 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 12 Dec 2010 12:55:53 -0700 Subject: [PATCH 15/15] Fix #7851 (some meta tags failed since V0.7.25) --- src/calibre/ebooks/html/input.py | 12 +++--------- src/calibre/ebooks/metadata/html.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index 059aeca324..6f875ae803 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -314,6 +314,8 @@ class HTMLInput(InputFormatPlugin): rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, \ xpath from calibre import guess_type + from calibre.ebooks.oeb.transforms.metadata import \ + meta_info_to_oeb_metadata import cssutils self.OEB_STYLES = OEB_STYLES oeb = create_oebbook(log, None, opts, self, @@ -321,15 +323,7 @@ class HTMLInput(InputFormatPlugin): self.oeb = oeb metadata = oeb.metadata - if mi.title: - metadata.add('title', mi.title) - if mi.authors: - for a in mi.authors: - metadata.add('creator', a, attrib={'role':'aut'}) - if mi.publisher: - metadata.add('publisher', mi.publisher) - if mi.isbn: - metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'}) + meta_info_to_oeb_metadata(mi, metadata, log) if not metadata.language: oeb.logger.warn(u'Language not specified') metadata.add('language', get_lang().replace('_', '-')) diff --git a/src/calibre/ebooks/metadata/html.py b/src/calibre/ebooks/metadata/html.py index f4eaa7cc61..fd42b2882f 100644 --- a/src/calibre/ebooks/metadata/html.py +++ b/src/calibre/ebooks/metadata/html.py @@ -170,7 +170,27 @@ def get_metadata_(src, encoding=None): if match: series = match.group(1) if series: + pat = re.compile(r'\[([.0-9]+)\]') + match = pat.search(series) + series_index = None + if match is not None: + try: + series_index = float(match.group(1)) + except: + pass + series = series.replace(match.group(), '').strip() + mi.series = ent_pat.sub(entity_to_unicode, series) + if series_index is None: + pat = get_meta_regexp_("Seriesnumber") + match = pat.search(src) + if match: + try: + series_index = float(match.group(1)) + except: + pass + if series_index is not None: + mi.series_index = series_index # RATING rating = None