From 0ebf1fa64cb1544ddc919383e4ca7ae1a3c63df7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Feb 2010 23:50:37 -0700 Subject: [PATCH 1/5] ... --- src/calibre/library/catalog.py | 10 +++++----- todo | 11 ----------- 2 files changed, 5 insertions(+), 16 deletions(-) delete mode 100644 todo diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 9555a69b0d..5c9d2e5161 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -3459,17 +3459,17 @@ class EPUB_MOBI(CatalogPlugin): if opts.connected_device['name']: if opts.connected_device['serial']: - build_log.append(" connected_device: '%s' #%s%s " % \ + build_log.append(u" connected_device: '%s' #%s%s " % \ (opts.connected_device['name'], opts.connected_device['serial'][0:4], 'x' * (len(opts.connected_device['serial']) - 4))) - build_log.append(" save_template: '%s'" % opts.connected_device['save_template']) + build_log.append(u" save_template: '%s'" % opts.connected_device['save_template']) else: - build_log.append(" connected_device: '%s'" % opts.connected_device['name']) + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) for storage in opts.connected_device['storage']: if storage: - build_log.append(" mount point: %s" % storage) - build_log.append(" save_template: '%s'" % opts.connected_device['save_template']) + build_log.append(u" mount point: %s" % storage) + build_log.append(u" save_template: '%s'" % opts.connected_device['save_template']) opts_dict = vars(opts) if opts_dict['ids']: diff --git a/todo b/todo deleted file mode 100644 index 7a639b25df..0000000000 --- a/todo +++ /dev/null @@ -1,11 +0,0 @@ - -* Refactor web.fetch.simple to use per connection timeouts via the timeout kwarg for mechanize.open - -* Testing framework - - -* Add a languages column to books. Best implementation is comma separated IANA codes -* Add a hash column to the formats table (used for stanza identifier) - -* Fix blockquote handling in sphinx templates -* Fix ebook-viewer going to links From 9182791f7353bf09d34b1e4ec569ed862231fe2b Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 23 Feb 2010 06:27:21 -0500 Subject: [PATCH 2/5] Fix bug #4979: pml to epub misinterprets \T behaviour. --- src/calibre/ebooks/pml/pmlconverter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/pml/pmlconverter.py b/src/calibre/ebooks/pml/pmlconverter.py index ed955879f8..1044dc9593 100644 --- a/src/calibre/ebooks/pml/pmlconverter.py +++ b/src/calibre/ebooks/pml/pmlconverter.py @@ -72,8 +72,8 @@ class PML_HTMLizer(object): 'ra': ('', ''), 'c': ('
', '
'), 'r': ('
', '
'), - 't': ('
', '
'), - 'T': ('
', '
'), + 't': ('
', '
'), + 'T': ('
', '
'), 'i': ('', ''), 'u': ('', ''), 'd': ('', ''), From 78c4467960533760fda01f7e8d4b394c2146bdee Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Feb 2010 09:26:31 -0700 Subject: [PATCH 3/5] Gamasutra by Darko Miletic --- resources/images/news/gamasutra_fa.png | Bin 0 -> 956 bytes resources/images/news/gamasutra_news.png | Bin 0 -> 956 bytes resources/recipes/gamasutra_fa.recipe | 56 +++++++++++++++++++++++ resources/recipes/gamasutra_news.recipe | 45 ++++++++++++++++++ src/calibre/ebooks/pdb/pdf/reader.py | 1 - 5 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 resources/images/news/gamasutra_fa.png create mode 100644 resources/images/news/gamasutra_news.png create mode 100644 resources/recipes/gamasutra_fa.recipe create mode 100644 resources/recipes/gamasutra_news.recipe diff --git a/resources/images/news/gamasutra_fa.png b/resources/images/news/gamasutra_fa.png new file mode 100644 index 0000000000000000000000000000000000000000..7d7c4eab6e29f35dc281bd60674d46dd041729fb GIT binary patch literal 956 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87?_@Vx;TbdoQ|D*K0l;X16@%WtS zgqD-9jQ5oN2<^KzUv=s0j9J@uH>oKpaePUA_HWOT>iGGJoHkY8^OY7EoM!iY&i3=m z-G0lU`E@Q1_bvAtyLcX%zQ2x}$x~wD$?GR(^)EgoVPdiWi>y-Df9Lz96IRUF{5ihh z={HjW5BJQX=t>~Eq}AT>;qq73bGkLIi-cTQ1Rc2-JLuhyd35O0cb#>5ix06p-sSzy zvF+eNMS+4{e2$t5c{5&QHs3s^y{GVt#Ch2nPSFvOQ-bH$ZkVmK^8W4~mcu?iJ`=no z8IBb8q(*Kz%dXPi{_Ewzo#l3ie*8R^8Xz}ohllz3gyT^HjRHotswf{wksbM$Mq z%3j}f>vWSUE~+J(JH0fU6;zSe8ogcT(o2)IVcfCTSB9;YXmgZkQMsg8RHd@c-#q5H zMExU%uxi$)$6c6Z@6^Z4I%{1q&wpWngUg28*=d_S&%b~4K6~LVL$}c6HAZd!{!G?C zDDWb+qnq*VC$`4-!YqtHb8m0km@9p}@W!gV2_i{BzZr$&Y%H8#{ZYSP6QS3Wl)L_7 zpzNWGa(i?8Gn@mw9`4eU{(gODbkyO>EMX&qFsbK%XBzLZ`ztmp$ctgy9<$|j6_bzd z*I&e$5jV@nnq8{Jm49kKW5vR6M(O>r3HKd7?$>*>>`qy3n!Wv@z4xqUF}tmKyQkDE zpx^Q6o%45YpXPb){;gzT|Dyv2>dX9^8;lNYPx^Cp#*Xt=mZh^EIlP>xC-LUsM`v}8 z#}j_;uRAg+bkma+{(eTrcBg@Y-!J_X*GtO?3VWl%0Zjd>C9V-ADTyViR>?)FK#IZ0 zz|cU~z)07~IK;rx$`FVwbPdd{3=IDM5!-^IAvZrIGp!O?gQ1a?iHVglP(MfmGs`?v Ppaup{S3j3^P616@%WtS zgqD-9jQ5oN2<^KzUv=s0j9J@uH>oKpaePUA_HWOT>iGGJoHkY8^OY7EoM!iY&i3=m z-G0lU`E@Q1_bvAtyLcX%zQ2x}$x~wD$?GR(^)EgoVPdiWi>y-Df9Lz96IRUF{5ihh z={HjW5BJQX=t>~Eq}AT>;qq73bGkLIi-cTQ1Rc2-JLuhyd35O0cb#>5ix06p-sSzy zvF+eNMS+4{e2$t5c{5&QHs3s^y{GVt#Ch2nPSFvOQ-bH$ZkVmK^8W4~mcu?iJ`=no z8IBb8q(*Kz%dXPi{_Ewzo#l3ie*8R^8Xz}ohllz3gyT^HjRHotswf{wksbM$Mq z%3j}f>vWSUE~+J(JH0fU6;zSe8ogcT(o2)IVcfCTSB9;YXmgZkQMsg8RHd@c-#q5H zMExU%uxi$)$6c6Z@6^Z4I%{1q&wpWngUg28*=d_S&%b~4K6~LVL$}c6HAZd!{!G?C zDDWb+qnq*VC$`4-!YqtHb8m0km@9p}@W!gV2_i{BzZr$&Y%H8#{ZYSP6QS3Wl)L_7 zpzNWGa(i?8Gn@mw9`4eU{(gODbkyO>EMX&qFsbK%XBzLZ`ztmp$ctgy9<$|j6_bzd z*I&e$5jV@nnq8{Jm49kKW5vR6M(O>r3HKd7?$>*>>`qy3n!Wv@z4xqUF}tmKyQkDE zpx^Q6o%45YpXPb){;gzT|Dyv2>dX9^8;lNYPx^Cp#*Xt=mZh^EIlP>xC-LUsM`v}8 z#}j_;uRAg+bkma+{(eTrcBg@Y-!J_X*GtO?3VWl%0Zjd>C9V-ADTyViR>?)FK#IZ0 zz|cU~z)07~IK;rx$`FVwbPdd{3=IDM5!-^IAvZrIGp!O?gQ1a?iHVglP(MfmGs`?v Ppaup{S3j3^P6.*?', re.DOTALL|re.IGNORECASE),lambda match: '<head><title>') + ,(re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '') + ] + remove_tags = [ + dict(name=['object','embed','iframe']) + ,dict(attrs={'class':'adBox'}) + ] + remove_tags_before = dict(attrs={'class':'title'}) + remove_attributes = ['width','height','name'] + + feeds = [(u'Feature Articles', u'http://feeds.feedburner.com/GamasutraFeatureArticles')] + + def print_version(self, url): + return url + '?print=1' + + def get_article_url(self, article): + return article.get('guid', None) + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) diff --git a/resources/recipes/gamasutra_news.recipe b/resources/recipes/gamasutra_news.recipe new file mode 100644 index 0000000000..ab7a089e1c --- /dev/null +++ b/resources/recipes/gamasutra_news.recipe @@ -0,0 +1,45 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +gamasutra.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Gamasutra(BasicNewsRecipe): + title = 'Gamasutra News' + __author__ = 'Darko Miletic' + description = 'The Art and Business of Making Games' + publisher = 'Gamasutra' + category = 'news, games, IT' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'cp1252' + use_embedded_content = False + language = 'en' + remove_empty_feeds = True + masthead_url = 'http://www.gamasutra.com/images/gamasutra_logo.gif' + extra_css = ' body{font-family: Verdana,Arial,Helvetica,sans-serif } img{margin-bottom: 0.4em} .newsTitle{font-size: xx-large; font-weight: bold} ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + remove_tags = [dict(attrs={'class':['relatedNews','adBox']})] + keep_only_tags = [dict(attrs={'class':['newsTitle','newsAuth','newsDate','newsText']})] + remove_attributes = ['width','height'] + + feeds = [(u'News', u'http://feeds.feedburner.com/GamasutraNews')] + + def get_article_url(self, article): + return article.get('guid', None) + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) diff --git a/src/calibre/ebooks/pdb/pdf/reader.py b/src/calibre/ebooks/pdb/pdf/reader.py index 913d06f634..7ad5776325 100644 --- a/src/calibre/ebooks/pdb/pdf/reader.py +++ b/src/calibre/ebooks/pdb/pdf/reader.py @@ -8,7 +8,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, John Schember ' __docformat__ = 'restructuredtext en' -import cStringIO from calibre.ebooks.pdb.formatreader import FormatReader from calibre.ptempfile import TemporaryFile From 1a4caa3801be0dbfdd7e5b69b0d3ee257ff728c9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Feb 2010 10:12:10 -0700 Subject: [PATCH 4/5] Move logic for removing inline navbars out of the BasicNewsRecipe class --- src/calibre/web/feeds/input.py | 5 +++++ src/calibre/web/feeds/news.py | 18 ++++++++---------- src/calibre/web/feeds/templates.py | 4 ++-- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/calibre/web/feeds/input.py b/src/calibre/web/feeds/input.py index adb2f13a56..a2527a2a70 100644 --- a/src/calibre/web/feeds/input.py +++ b/src/calibre/web/feeds/input.py @@ -101,5 +101,10 @@ class RecipeInput(InputFormatPlugin): return os.path.abspath(f) def postprocess_book(self, oeb, opts, log): + if opts.no_inline_navbars: + from calibre.ebooks.oeb.base import XPath + for item in oeb.spine: + for div in XPath('//h:div[contains(@class, "calibre_navbar")]')(item.data): + div.getparent().remove(div) self.recipe_object.postprocess_book(oeb, opts, log) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 8b1757371d..d0c9d941e3 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -266,7 +266,7 @@ class BasicNewsRecipe(Recipe): font-weight: bold; } - .navbar { + .calibre_navbar { font-family:monospace; } ''' @@ -525,7 +525,6 @@ class BasicNewsRecipe(Recipe): self.username = options.username self.password = options.password self.lrf = options.lrf - self.include_navbars = not options.no_inline_navbars self.output_dir = os.path.abspath(self.output_dir) if options.test: @@ -597,7 +596,7 @@ class BasicNewsRecipe(Recipe): if first_fetch and job_info: url, f, a, feed_len = job_info body = soup.find('body') - if body is not None and self.include_navbars: + if body is not None: templ = self.navbar.generate(False, f, a, feed_len, not self.has_single_feed, url, __appname__, @@ -1149,13 +1148,12 @@ class BasicNewsRecipe(Recipe): body = soup.find('body') if body is not None: prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - if self.include_navbars: - templ = self.navbar.generate(True, num, j, len(f), - not self.has_single_feed, - a.orig_url, __appname__, prefix=prefix, - center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) + templ = self.navbar.generate(True, num, j, len(f), + not self.has_single_feed, + a.orig_url, __appname__, prefix=prefix, + center=self.center_navbar) + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) with open(last, 'wb') as fi: fi.write(unicode(soup).encode('utf-8')) diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index e254b6eded..954677a90e 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -38,7 +38,7 @@ class NavBarTemplate(Template): -
+

This article was downloaded by ${__appname__} from ${url} @@ -167,7 +167,7 @@ class FeedTemplate(Template): -

+
| Up one level |
From 180308c7ce26308dc6c2665bd0213a31d0083d31 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 23 Feb 2010 12:33:25 -0700 Subject: [PATCH 5/5] News download: When using the debug pipeline options, create a zip file named periodical.downloaded_recipe in the debug directory. This can be passed to ebook-convert to directly convert a previous download into an e-book. --- src/calibre/customize/conversion.py | 13 ++++ src/calibre/ebooks/conversion/plumber.py | 11 ++- src/calibre/web/feeds/input.py | 97 ++++++++++++++---------- 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index bd79f85e7f..6fd3fb9932 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -214,8 +214,21 @@ class InputFormatPlugin(Plugin): return ret def postprocess_book(self, oeb, opts, log): + ''' + Called to allow the input plugin to perform postprocessing after + the book has been parsed. + ''' pass + def specialize(self, oeb, opts, log, output_fmt): + ''' + Called to allow the input plugin to specialize the parsed book + for a particular output format. Called after postprocess_book + and before any transforms are performed on the parsed book. + ''' + pass + + class OutputFormatPlugin(Plugin): ''' OutputFormatPlugins are responsible for converting an OEB document diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index b6e7684c48..5c8e34c139 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -13,6 +13,7 @@ from calibre.customize.ui import input_profiles, output_profiles, \ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.date import parse_date +from calibre.utils.zipfile import ZipFile from calibre import extract, walk DEBUG_README=u''' @@ -726,6 +727,12 @@ OptionRecommendation(name='timestamp', else: os.makedirs(out_dir) self.dump_oeb(ret, out_dir) + if self.input_fmt == 'recipe': + zf = ZipFile(os.path.join(self.opts.debug_pipeline, + 'periodical.downloaded_recipe'), 'w') + zf.add_dir(out_dir) + self.input_plugin.save_download(zf) + zf.close() self.log.info('Input debug saved to:', out_dir) @@ -780,7 +787,7 @@ OptionRecommendation(name='timestamp', self.dump_input(self.oeb, tdir) if self.abort_after_input_dump: return - if self.input_fmt == 'recipe': + if self.input_fmt in ('recipe', 'downloaded_recipe'): self.opts_to_mi(self.user_metadata) if not hasattr(self.oeb, 'manifest'): self.oeb = create_oebbook(self.log, self.oeb, self.opts, @@ -793,6 +800,8 @@ OptionRecommendation(name='timestamp', out_dir = os.path.join(self.opts.debug_pipeline, 'parsed') self.dump_oeb(self.oeb, out_dir) self.log('Parsed HTML written to:', out_dir) + self.input_plugin.specialize(self.oeb, self.opts, self.log, + self.output_fmt) pr(0., _('Running transforms on ebook...')) diff --git a/src/calibre/web/feeds/input.py b/src/calibre/web/feeds/input.py index a2527a2a70..51aaeb3e4b 100644 --- a/src/calibre/web/feeds/input.py +++ b/src/calibre/web/feeds/input.py @@ -19,7 +19,7 @@ class RecipeInput(InputFormatPlugin): name = 'Recipe Input' author = 'Kovid Goyal' description = _('Download periodical content from the internet') - file_types = set(['recipe']) + file_types = set(['recipe', 'downloaded_recipe']) recommendations = set([ ('chapter', None, OptionRecommendation.HIGH), @@ -51,60 +51,79 @@ class RecipeInput(InputFormatPlugin): def convert(self, recipe_or_file, opts, file_ext, log, accelerators): from calibre.web.feeds.recipes import compile_recipe - from calibre.web.feeds.recipes.collection import \ - get_builtin_recipe_by_title - if os.access(recipe_or_file, os.R_OK): - recipe = compile_recipe(open(recipe_or_file, 'rb').read()) + opts.output_profile.flow_size = 0 + if file_ext == 'downloaded_recipe': + from calibre.utils.zipfile import ZipFile + zf = ZipFile(recipe_or_file, 'r') + zf.extractall() + zf.close() + self.recipe_source = open('download.recipe', 'rb').read() + recipe = compile_recipe(self.recipe_source) + self.recipe_object = recipe(opts, log, self.report_progress) else: - title = getattr(opts, 'original_recipe_input_arg', recipe_or_file) - title = os.path.basename(title).rpartition('.')[0] - raw = get_builtin_recipe_by_title(title, log=log, - download_recipe=not opts.dont_download_recipe) - builtin = False - try: - recipe = compile_recipe(raw) - if recipe.requires_version > numeric_version: - log.warn( - 'Downloaded recipe needs calibre version at least: %s' % \ - ('.'.join(recipe.requires_version))) - builtin = True - except: - log.exception('Failed to compile downloaded recipe. Falling ' - 'back to builtin one') - builtin = True - if builtin: + if os.access(recipe_or_file, os.R_OK): + self.recipe_source = open(recipe_or_file, 'rb').read() + recipe = compile_recipe(self.recipe_source) + else: + from calibre.web.feeds.recipes.collection import \ + get_builtin_recipe_by_title + title = getattr(opts, 'original_recipe_input_arg', recipe_or_file) + title = os.path.basename(title).rpartition('.')[0] raw = get_builtin_recipe_by_title(title, log=log, - download_recipe=False) - if raw is None: - raise ValueError('Failed to find builtin recipe: '+title) - recipe = compile_recipe(raw) + download_recipe=not opts.dont_download_recipe) + builtin = False + try: + recipe = compile_recipe(raw) + self.recipe_source = raw + if recipe.requires_version > numeric_version: + log.warn( + 'Downloaded recipe needs calibre version at least: %s' % \ + ('.'.join(recipe.requires_version))) + builtin = True + except: + log.exception('Failed to compile downloaded recipe. Falling ' + 'back to builtin one') + builtin = True + if builtin: + raw = get_builtin_recipe_by_title(title, log=log, + download_recipe=False) + if raw is None: + raise ValueError('Failed to find builtin recipe: '+title) + recipe = compile_recipe(raw) + self.recipe_source = raw + if recipe is None: + raise ValueError('%r is not a valid recipe file or builtin recipe' % + recipe_or_file) + ro = recipe(opts, log, self.report_progress) + disabled = getattr(ro, 'recipe_disabled', None) + if disabled is not None: + raise RecipeDisabled(disabled) + ro.download() + self.recipe_object = ro - if recipe is None: - raise ValueError('%r is not a valid recipe file or builtin recipe' % - recipe_or_file) - - ro = recipe(opts, log, self.report_progress) - disabled = getattr(ro, 'recipe_disabled', None) - if disabled is not None: - raise RecipeDisabled(disabled) - ro.download() - self.recipe_object = ro for key, val in recipe.conversion_options.items(): setattr(opts, key, val) - opts.output_profile.flow_size = 0 - for f in os.listdir('.'): if f.endswith('.opf'): return os.path.abspath(f) def postprocess_book(self, oeb, opts, log): + if self.recipe_object is not None: + self.recipe_object.postprocess_book(oeb, opts, log) + + def specialize(self, oeb, opts, log, output_fmt): if opts.no_inline_navbars: from calibre.ebooks.oeb.base import XPath for item in oeb.spine: for div in XPath('//h:div[contains(@class, "calibre_navbar")]')(item.data): div.getparent().remove(div) - self.recipe_object.postprocess_book(oeb, opts, log) + + def save_download(self, zf): + raw = self.recipe_source + if isinstance(raw, unicode): + raw = raw.encode('utf-8') + zf.writestr('download.recipe', raw)