diff --git a/resources/images/news/gamasutra_fa.png b/resources/images/news/gamasutra_fa.png new file mode 100644 index 0000000000..7d7c4eab6e Binary files /dev/null and b/resources/images/news/gamasutra_fa.png differ diff --git a/resources/images/news/gamasutra_news.png b/resources/images/news/gamasutra_news.png new file mode 100644 index 0000000000..7d7c4eab6e Binary files /dev/null and b/resources/images/news/gamasutra_news.png differ diff --git a/resources/recipes/gamasutra_fa.recipe b/resources/recipes/gamasutra_fa.recipe new file mode 100644 index 0000000000..26a56849f7 --- /dev/null +++ b/resources/recipes/gamasutra_fa.recipe @@ -0,0 +1,56 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +gamasutra.com +''' + +import re +from calibre.web.feeds.news import BasicNewsRecipe + +class Gamasutra(BasicNewsRecipe): + title = 'Gamasutra Featured articles' + __author__ = 'Darko Miletic' + description = 'The Art and Business of Making Games' + publisher = 'Gamasutra' + category = 'news, games, IT' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'cp1252' + use_embedded_content = False + language = 'en' + remove_empty_feeds = True + masthead_url = 'http://www.gamasutra.com/images/gamasutra_logo.gif' + extra_css = ' body{font-family: Verdana,Arial,Helvetica,sans-serif } img{margin-bottom: 0.4em} .title{font-size: x-large; font-weight: bold} ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + preprocess_regexps = [ + (re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '<head><title>') + ,(re.compile(r'.*?', re.DOTALL|re.IGNORECASE),lambda match: '') + ,(re.compile(r'', re.DOTALL|re.IGNORECASE),lambda match: '') + ] + remove_tags = [ + dict(name=['object','embed','iframe']) + ,dict(attrs={'class':'adBox'}) + ] + remove_tags_before = dict(attrs={'class':'title'}) + remove_attributes = ['width','height','name'] + + feeds = [(u'Feature Articles', u'http://feeds.feedburner.com/GamasutraFeatureArticles')] + + def print_version(self, url): + return url + '?print=1' + + def get_article_url(self, article): + return article.get('guid', None) + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) diff --git a/resources/recipes/gamasutra_news.recipe b/resources/recipes/gamasutra_news.recipe new file mode 100644 index 0000000000..ab7a089e1c --- /dev/null +++ b/resources/recipes/gamasutra_news.recipe @@ -0,0 +1,45 @@ +__license__ = 'GPL v3' +__copyright__ = '2010, Darko Miletic ' +''' +gamasutra.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class Gamasutra(BasicNewsRecipe): + title = 'Gamasutra News' + __author__ = 'Darko Miletic' + description = 'The Art and Business of Making Games' + publisher = 'Gamasutra' + category = 'news, games, IT' + oldest_article = 2 + max_articles_per_feed = 200 + no_stylesheets = True + encoding = 'cp1252' + use_embedded_content = False + language = 'en' + remove_empty_feeds = True + masthead_url = 'http://www.gamasutra.com/images/gamasutra_logo.gif' + extra_css = ' body{font-family: Verdana,Arial,Helvetica,sans-serif } img{margin-bottom: 0.4em} .newsTitle{font-size: xx-large; font-weight: bold} ' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'linearize_tables' : True + } + + remove_tags = [dict(attrs={'class':['relatedNews','adBox']})] + keep_only_tags = [dict(attrs={'class':['newsTitle','newsAuth','newsDate','newsText']})] + remove_attributes = ['width','height'] + + feeds = [(u'News', u'http://feeds.feedburner.com/GamasutraNews')] + + def get_article_url(self, article): + return article.get('guid', None) + + def preprocess_html(self, soup): + for item in soup.findAll(style=True): + del item['style'] + return self.adeify_images(soup) diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py index bd79f85e7f..6fd3fb9932 100644 --- a/src/calibre/customize/conversion.py +++ b/src/calibre/customize/conversion.py @@ -214,8 +214,21 @@ class InputFormatPlugin(Plugin): return ret def postprocess_book(self, oeb, opts, log): + ''' + Called to allow the input plugin to perform postprocessing after + the book has been parsed. + ''' pass + def specialize(self, oeb, opts, log, output_fmt): + ''' + Called to allow the input plugin to specialize the parsed book + for a particular output format. Called after postprocess_book + and before any transforms are performed on the parsed book. + ''' + pass + + class OutputFormatPlugin(Plugin): ''' OutputFormatPlugins are responsible for converting an OEB document diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index b6e7684c48..5c8e34c139 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -13,6 +13,7 @@ from calibre.customize.ui import input_profiles, output_profiles, \ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.date import parse_date +from calibre.utils.zipfile import ZipFile from calibre import extract, walk DEBUG_README=u''' @@ -726,6 +727,12 @@ OptionRecommendation(name='timestamp', else: os.makedirs(out_dir) self.dump_oeb(ret, out_dir) + if self.input_fmt == 'recipe': + zf = ZipFile(os.path.join(self.opts.debug_pipeline, + 'periodical.downloaded_recipe'), 'w') + zf.add_dir(out_dir) + self.input_plugin.save_download(zf) + zf.close() self.log.info('Input debug saved to:', out_dir) @@ -780,7 +787,7 @@ OptionRecommendation(name='timestamp', self.dump_input(self.oeb, tdir) if self.abort_after_input_dump: return - if self.input_fmt == 'recipe': + if self.input_fmt in ('recipe', 'downloaded_recipe'): self.opts_to_mi(self.user_metadata) if not hasattr(self.oeb, 'manifest'): self.oeb = create_oebbook(self.log, self.oeb, self.opts, @@ -793,6 +800,8 @@ OptionRecommendation(name='timestamp', out_dir = os.path.join(self.opts.debug_pipeline, 'parsed') self.dump_oeb(self.oeb, out_dir) self.log('Parsed HTML written to:', out_dir) + self.input_plugin.specialize(self.oeb, self.opts, self.log, + self.output_fmt) pr(0., _('Running transforms on ebook...')) diff --git a/src/calibre/ebooks/pdb/pdf/reader.py b/src/calibre/ebooks/pdb/pdf/reader.py index 913d06f634..7ad5776325 100644 --- a/src/calibre/ebooks/pdb/pdf/reader.py +++ b/src/calibre/ebooks/pdb/pdf/reader.py @@ -8,7 +8,6 @@ __license__ = 'GPL v3' __copyright__ = '2010, John Schember ' __docformat__ = 'restructuredtext en' -import cStringIO from calibre.ebooks.pdb.formatreader import FormatReader from calibre.ptempfile import TemporaryFile diff --git a/src/calibre/ebooks/pml/pmlconverter.py b/src/calibre/ebooks/pml/pmlconverter.py index ed955879f8..1044dc9593 100644 --- a/src/calibre/ebooks/pml/pmlconverter.py +++ b/src/calibre/ebooks/pml/pmlconverter.py @@ -72,8 +72,8 @@ class PML_HTMLizer(object): 'ra': ('', ''), 'c': ('
', '
'), 'r': ('
', '
'), - 't': ('
', '
'), - 'T': ('
', '
'), + 't': ('
', '
'), + 'T': ('
', '
'), 'i': ('', ''), 'u': ('', ''), 'd': ('', ''), diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 39fb503636..7271d5af88 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -866,18 +866,24 @@ class EPUB_MOBI(CatalogPlugin): return property(fget=fget, fset=fset) @dynamic_property - def READ_SYMBOL(self): - def fget(self): - return '✓' if self.generateForKindle else \ - '%s' % self.opts.read_tag - return property(fget=fget) - @dynamic_property def NOT_READ_SYMBOL(self): def fget(self): return '✓' if self.generateForKindle else \ '%s' % self.opts.read_tag return property(fget=fget) @dynamic_property + def READING_SYMBOL(self): + def fget(self): + return '▷' if self.generateForKindle else \ + '%s' % self.opts.read_tag + return property(fget=fget) + @dynamic_property + def READ_SYMBOL(self): + def fget(self): + return '✓' if self.generateForKindle else \ + '%s' % self.opts.read_tag + return property(fget=fget) + @dynamic_property def FULL_RATING_SYMBOL(self): def fget(self): return "★" if self.generateForKindle else "*" @@ -1196,7 +1202,6 @@ class EPUB_MOBI(CatalogPlugin): self.timestamp, = unpack('>I', data[0x24:0x28]) bpar_offset, = unpack('>I', data[0x4e:0x52]) #print "bpar_offset: 0x%x" % bpar_offset - bpl = bpar_offset + 0x04 lrlo = bpar_offset + 0x0c self.last_read_location = int(unpack('>I', data[lrlo:lrlo+4])[0]) ''' @@ -1285,7 +1290,9 @@ class EPUB_MOBI(CatalogPlugin): d.initialize(self.opts.connected_device['save_template']) bookmarks = {} for book in self.booksByTitle: - myMeta = MetaInformation(book['title'], + original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ + else book['title'] + myMeta = MetaInformation(original_title, authors=book['authors']) myMeta.author_sort = book['author_sort'] bm_found = False @@ -1550,12 +1557,16 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # Prefix book with read/unread symbol + # book with read/reading/unread symbol if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) pBookTag['class'] = "read_book" ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 else: # hidden check mark pBookTag['class'] = "unread_book" @@ -1704,12 +1715,16 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read/unread symbol + # book with read/reading/unread symbol if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) pBookTag['class'] = "read_book" ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 else: # hidden check mark pBookTag['class'] = "unread_book" @@ -1817,12 +1832,16 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # Prefix book with read/unread symbol + # book with read/reading/unread symbol if new_entry['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) pBookTag['class'] = "read_book" ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 else: # hidden check mark pBookTag['class'] = "unread_book" @@ -1859,12 +1878,16 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # Prefix book with read/unread symbol + # book with read/reading/unread symbol if new_entry['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) pBookTag['class'] = "read_book" ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 else: # hidden check mark pBookTag['class'] = "unread_book" @@ -3529,14 +3552,21 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # Prefix book with read/unread symbol + # book with read/reading/unread symbol if book['read']: + # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 else: + # hidden check mark pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 + ptc += 1 # Add the book title aTag = Tag(soup, "a") @@ -4010,18 +4040,18 @@ class EPUB_MOBI(CatalogPlugin): if opts.connected_device['name']: if opts.connected_device['serial']: - build_log.append(" connected_device: '%s' #%s%s " % \ + build_log.append(u" connected_device: '%s' #%s%s " % \ (opts.connected_device['name'], opts.connected_device['serial'][0:4], 'x' * (len(opts.connected_device['serial']) - 4))) for storage in opts.connected_device['storage']: if storage: - build_log.append(" mount point: %s" % storage) + build_log.append(u" mount point: %s" % storage) else: - build_log.append(" connected_device: '%s'" % opts.connected_device['name']) + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) for storage in opts.connected_device['storage']: if storage: - build_log.append(" mount point: %s" % storage) + build_log.append(u" mount point: %s" % storage) opts_dict = vars(opts) if opts_dict['ids']: diff --git a/src/calibre/web/feeds/input.py b/src/calibre/web/feeds/input.py index adb2f13a56..51aaeb3e4b 100644 --- a/src/calibre/web/feeds/input.py +++ b/src/calibre/web/feeds/input.py @@ -19,7 +19,7 @@ class RecipeInput(InputFormatPlugin): name = 'Recipe Input' author = 'Kovid Goyal' description = _('Download periodical content from the internet') - file_types = set(['recipe']) + file_types = set(['recipe', 'downloaded_recipe']) recommendations = set([ ('chapter', None, OptionRecommendation.HIGH), @@ -51,55 +51,79 @@ class RecipeInput(InputFormatPlugin): def convert(self, recipe_or_file, opts, file_ext, log, accelerators): from calibre.web.feeds.recipes import compile_recipe - from calibre.web.feeds.recipes.collection import \ - get_builtin_recipe_by_title - if os.access(recipe_or_file, os.R_OK): - recipe = compile_recipe(open(recipe_or_file, 'rb').read()) + opts.output_profile.flow_size = 0 + if file_ext == 'downloaded_recipe': + from calibre.utils.zipfile import ZipFile + zf = ZipFile(recipe_or_file, 'r') + zf.extractall() + zf.close() + self.recipe_source = open('download.recipe', 'rb').read() + recipe = compile_recipe(self.recipe_source) + self.recipe_object = recipe(opts, log, self.report_progress) else: - title = getattr(opts, 'original_recipe_input_arg', recipe_or_file) - title = os.path.basename(title).rpartition('.')[0] - raw = get_builtin_recipe_by_title(title, log=log, - download_recipe=not opts.dont_download_recipe) - builtin = False - try: - recipe = compile_recipe(raw) - if recipe.requires_version > numeric_version: - log.warn( - 'Downloaded recipe needs calibre version at least: %s' % \ - ('.'.join(recipe.requires_version))) - builtin = True - except: - log.exception('Failed to compile downloaded recipe. Falling ' - 'back to builtin one') - builtin = True - if builtin: + if os.access(recipe_or_file, os.R_OK): + self.recipe_source = open(recipe_or_file, 'rb').read() + recipe = compile_recipe(self.recipe_source) + else: + from calibre.web.feeds.recipes.collection import \ + get_builtin_recipe_by_title + title = getattr(opts, 'original_recipe_input_arg', recipe_or_file) + title = os.path.basename(title).rpartition('.')[0] raw = get_builtin_recipe_by_title(title, log=log, - download_recipe=False) - if raw is None: - raise ValueError('Failed to find builtin recipe: '+title) - recipe = compile_recipe(raw) + download_recipe=not opts.dont_download_recipe) + builtin = False + try: + recipe = compile_recipe(raw) + self.recipe_source = raw + if recipe.requires_version > numeric_version: + log.warn( + 'Downloaded recipe needs calibre version at least: %s' % \ + ('.'.join(recipe.requires_version))) + builtin = True + except: + log.exception('Failed to compile downloaded recipe. Falling ' + 'back to builtin one') + builtin = True + if builtin: + raw = get_builtin_recipe_by_title(title, log=log, + download_recipe=False) + if raw is None: + raise ValueError('Failed to find builtin recipe: '+title) + recipe = compile_recipe(raw) + self.recipe_source = raw + if recipe is None: + raise ValueError('%r is not a valid recipe file or builtin recipe' % + recipe_or_file) + ro = recipe(opts, log, self.report_progress) + disabled = getattr(ro, 'recipe_disabled', None) + if disabled is not None: + raise RecipeDisabled(disabled) + ro.download() + self.recipe_object = ro - if recipe is None: - raise ValueError('%r is not a valid recipe file or builtin recipe' % - recipe_or_file) - - ro = recipe(opts, log, self.report_progress) - disabled = getattr(ro, 'recipe_disabled', None) - if disabled is not None: - raise RecipeDisabled(disabled) - ro.download() - self.recipe_object = ro for key, val in recipe.conversion_options.items(): setattr(opts, key, val) - opts.output_profile.flow_size = 0 - for f in os.listdir('.'): if f.endswith('.opf'): return os.path.abspath(f) def postprocess_book(self, oeb, opts, log): - self.recipe_object.postprocess_book(oeb, opts, log) + if self.recipe_object is not None: + self.recipe_object.postprocess_book(oeb, opts, log) + + def specialize(self, oeb, opts, log, output_fmt): + if opts.no_inline_navbars: + from calibre.ebooks.oeb.base import XPath + for item in oeb.spine: + for div in XPath('//h:div[contains(@class, "calibre_navbar")]')(item.data): + div.getparent().remove(div) + + def save_download(self, zf): + raw = self.recipe_source + if isinstance(raw, unicode): + raw = raw.encode('utf-8') + zf.writestr('download.recipe', raw) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 8b1757371d..d0c9d941e3 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -266,7 +266,7 @@ class BasicNewsRecipe(Recipe): font-weight: bold; } - .navbar { + .calibre_navbar { font-family:monospace; } ''' @@ -525,7 +525,6 @@ class BasicNewsRecipe(Recipe): self.username = options.username self.password = options.password self.lrf = options.lrf - self.include_navbars = not options.no_inline_navbars self.output_dir = os.path.abspath(self.output_dir) if options.test: @@ -597,7 +596,7 @@ class BasicNewsRecipe(Recipe): if first_fetch and job_info: url, f, a, feed_len = job_info body = soup.find('body') - if body is not None and self.include_navbars: + if body is not None: templ = self.navbar.generate(False, f, a, feed_len, not self.has_single_feed, url, __appname__, @@ -1149,13 +1148,12 @@ class BasicNewsRecipe(Recipe): body = soup.find('body') if body is not None: prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last)))) - if self.include_navbars: - templ = self.navbar.generate(True, num, j, len(f), - not self.has_single_feed, - a.orig_url, __appname__, prefix=prefix, - center=self.center_navbar) - elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') - body.insert(len(body.contents), elem) + templ = self.navbar.generate(True, num, j, len(f), + not self.has_single_feed, + a.orig_url, __appname__, prefix=prefix, + center=self.center_navbar) + elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div') + body.insert(len(body.contents), elem) with open(last, 'wb') as fi: fi.write(unicode(soup).encode('utf-8')) diff --git a/src/calibre/web/feeds/templates.py b/src/calibre/web/feeds/templates.py index e254b6eded..954677a90e 100644 --- a/src/calibre/web/feeds/templates.py +++ b/src/calibre/web/feeds/templates.py @@ -38,7 +38,7 @@ class NavBarTemplate(Template): -
+

This article was downloaded by ${__appname__} from ${url} @@ -167,7 +167,7 @@ class FeedTemplate(Template): -

+
| Up one level |
diff --git a/todo b/todo deleted file mode 100644 index 7a639b25df..0000000000 --- a/todo +++ /dev/null @@ -1,11 +0,0 @@ - -* Refactor web.fetch.simple to use per connection timeouts via the timeout kwarg for mechanize.open - -* Testing framework - - -* Add a languages column to books. Best implementation is comma separated IANA codes -* Add a hash column to the formats table (used for stanza identifier) - -* Fix blockquote handling in sphinx templates -* Fix ebook-viewer going to links