merge from trunk

.bzrignore

@@ -3,9 +3,9 @@
 src/calibre/plugins
 resources/images.qrc
 src/calibre/ebooks/oeb/display/test/*.js
-src/calibre/manual/.build/
-src/calibre/manual/cli/
-src/calibre/manual/template_ref.rst
+manual/.build/
+manual/cli/
+manual/template_ref.rst
 build
 dist
 docs

COPYRIGHT

@@ -4,11 +4,6 @@ License: GPL-3
  The full text of the GPL is distributed as in
  /usr/share/common-licenses/GPL-3 on Debian systems.
 
-Files: src/calibre/ebooks/pdf/*.h,*.cpp
-License: GPL-2 or later
- The full text of the GPL is distributed as in
- /usr/share/common-licenses/GPL-2 on Debian systems.
-
 Files: setup/iso_639/*
 Copyright: Various
 License: LGPL 2.1
@@ -21,6 +16,12 @@ License: BSD
  The full text of the BSD license is distributed as in
  /usr/share/common-licenses/BSD on Debian systems.
 
+Files: src/qtcurve/*
+Copyright: Craig Drummond, 2007 - 2010 craig.p.drummond@gmail.com
+License: GPL-2
+ The full text of the GPL is distributed as in
+ /usr/share/common-licenses/GPL-2 on Debian systems.
+
 Files: src/calibre/ebooks/chardet/*
 Copyright: Copyright (C) 1998-2001 Netscape Communications Corporation
 License: LGPL-2.1+

Changelog.yaml

@@ -19,6 +19,231 @@
 #   new recipes:
 #     - title: 
 
+
+- version: 0.8.56
+  date: 2012-06-15
+
+  new features:
+    - title: "Make the new calibre style default on Windows and OS X."
+      type: major
+      description: "This change gives a more 'modern' feel to the calibre user interface with focus highlighting, gradients, rounded corners, etc. In case you prefer the old look, you can restore under Preferences->Look & Feel->User interface style"
+
+    - title: "Get Books: Add the new SONY Reader store"
+
+    - title: "Read metadata from .docx (Microsoft Word) files"
+
+    - title: "Allow customizing the behavior of the searching for similar books by right clicking the book. You can now tell calibre to search different columns than the traditional author/series/publisher/tags/etc. in Preferences->Searching"
+
+    - title: "Add option to restore alternating row colors to the Tag Browser under Preferences->Look & Feel->Tag Browser"
+
+    - title: "Update to Qt 4.8.2 on windows compiled with link time code generation for a small performance boost"
+ 
+  bug fixes:
+    - title: "Get Books: Update plugins to handle website changes at ebooks.com, project gutenberg, and virtualo"
+
+    - title: "AZW3 Output: Fix TOC at start option not working"
+
+    - title: "AZW3 Output: Close self closing script/style/title/head tags explicitly as they cause problems in webkit based renderers like the Kindle Fire and calibre's viewers."
+
+    - title: "Fix the current_library_name() template function not updating after a library switch"
+
+    - title: "AZW3 Output: Handle the case of a link pointing to the last line of text in the document."
+      tickets: [1011330]
+
+    - title: "Fix regression in 0.8.55 that broke highlighting of items matching a search in the Tag Browser"
+      tickets: [1011030] 
+
+    - title: "News download: Handle query only relative URLs"
+
+  improved recipes:
+    - Christian Science Monitor
+    - Neue Zurcher Zeitung
+    - Birmignham Post
+    - Metro UK 
+    - New Musical Express
+    - The Independent
+    - The Daily Mirror
+    - Vreme
+    - Smithsonian Magazine
+
+  new recipes:
+    - title: NZZ Webpaper 
+      author: Bernd Leinfelder
+
+
+- version: 0.8.55
+  date: 2012-06-08
+
+  new features:
+    - title: "Add a new 'Calibre style' interface look that is more modern than the default look. You can select it via Preferences->Look & Feel->User interface style."
+
+    - title: "New, subtler look for the Tag Browser"
+
+    - title: "Driver for Trekstor Pyrus and Pantech Android Tablet"
+      tickets: [1008946, 1007929]
+
+    - title: "Conversion pipeline: Handle guide elements with incorrectly cased hrefs. Also handle guide elements of type coverimagestandard and thumbimagestandard."
+
+    - title: "Allow user to customize trekstor plugin to send books into sub directories."
+      tickets: [1007646]
+
+    - title: "EPUB Input: Add support for EPUB files that use the IDPF font obfuscation algorithm. Apparently, people have started producing these now."
+      tickets: [1008810]
+
+    - title: "Save single format to disk: Only show the format available in the selected books."
+      tickets: [1007287]
+ 
+  bug fixes:
+    - title: "MOBI Output: When using the insert metadata at start of book option, do not use a table to layout the metadata, as the Kindle Fire crashes when rendering the table."
+      tickets: [1002119]
+
+    - title: "Device detection: Fix a bug that could cause device detection to fail completely if devices with certain vendor/product ids are connected."
+      tickets: [1009718]
+
+    - title: "MOBI Output: When rasterizing svgs only compute style information when an actual svg image is present. Small speedup when converting large svg-free documents to MOBI."
+
+    - title: "SONY T1 driver: Fix support for collections of books placed on the SD card"
+      tickets: [986044]
+
+    - title: "Fix partitioning problems in tag browser with fields that have no name, such as identifiers and formats"
+
+    - title: "Welcome wizard: Preferentially use the kindle email address set as default when more than one such address exists."
+      tickets: [1007932 ]
+
+    - title: "Fix regression in 0.8.54 that broke the use of the shortcut Alt+A to select books by the same author"
+
+  improved recipes:
+    - Various Polish recipes
+    - Vice Magazine
+    - EL Mundo Today
+    - Haaretz
+    - Good Housekeeping
+    - El Pais
+    - Christian Science Monitor
+    - Marketing Magazine
+    - Instapaper
+
+  new recipes:
+    - title: Various Philippine news sources 
+      author: jde
+
+    - title: Natemat.pl and wirtualnemedia.pl 
+      author: fenuks
+
+    - title: Rabble.ca
+      author: timtoo
+
+- version: 0.8.54
+  date: 2012-05-31
+
+  new features:
+    - title: "E-book viewer: The Table of contents panel now tracks the current position in the book. As you scroll through the book, the entry you are currently on is highlighted."
+      type: major
+      description: "To see this feature in action, open the Table of Contents panel in the viewer by clicking the button with three blue lines on it. As you page through the book, the chapter you are reading currently is highlighted in the Table of Contents Panel. Obviously, this will only work if the book you are reading has a Table of Contents. You can also use the Ctrl+PgUp and Ctrl+PgDn keys to quickly skip between chapters."
+
+    - title: "calibredb: Allow setting metadata for individual fields with the set_metadata command"
+
+    - title: "Make it a little harder to accidentally change the sorting of items in the Tag Browser. Also frees up more vertical space for the Tag Browser itself."
+
+    - title: "The calibre user manual is now available in AZW3 format as well as EPUB"
+
+  bug fixes:
+    - title: "Automatic titlecasing: No longer try to capitalize scottish names, as there are too many special cases." 
+      tickets: [775825] 
+
+    - title: "Never crash when reading metadata from PDF files (reading now always happens in a worker process)"
+      tickets: [1006452]
+
+    - title: "EPUB Input: Do no skip the valid children of an NCX node that has no text/href"
+
+    - title: "Archos driver: Detect SD card"
+      tickets: [1005650] 
+
+    - title: "When bulk downloading metadata and the user deletes one of the books for which metadata is being downloaded, just ignore it, instead of erroring out"
+
+    - title: "When deleting books from the bottom of the booklist, ensure that the bottom book after deleting is selected"
+
+    - title: "Fix regression in 0.8.53 that broke sending APNX files to older Kindle devices"
+
+    - title: "Use correct text color for selected rows in the list of matches when downloading metadata and showing results in get books."
+      tickets: [1004568]
+
+  improved recipes:
+    - The Independent
+    - Welt der Physik
+    - China Daily
+    - The Grid
+    - Prospect Magazine
+
+  new recipes:
+    - title: La gazetta del Mezzogiorno 
+      author: faber1971
+
+- version: 0.8.53
+  date: 2012-05-25
+
+  new features:
+    - title: "Kindle Touch/4 driver: Upload cover thumbnails when sending books to device by USB to workaround Amazon bug of not displaying covers for sync-enabled books"
+
+    - title: "Support for updating metadata in FB2 files"
+
+    - title: "Set a different background color when choosing formats to not delete as opposed to choosing format to delete."
+      tickets: [ 1001741 ] 
+
+    - title: "E-book viewer: Add an option to prevent the up and down arrow keys from scrolling past page breaks"
+
+    - title: "Get Books: Remove ebookshoppe.com at the website's request"
+ 
+  bug fixes:
+    - title: "PDF Input: Support image rotation commands in PDF files. Fixes the long standing problem of some  images being flipped when converting from PDF in calibre."
+
+    - title: "Fix a regression in 0.8.51 that caused conversion to HTMLZ to not have any CSS"
+
+    - title: "Get Books: Fix website change at kobo.com causing prices not to be found"
+
+    - title: "Edit the time in the 24 hour clock when calibre's interface language is set to German."
+      tickets: [ 1001809 ] 
+
+    - title: "MOBI Output: When generating joint KF8/MOBI6 .mobi files set the text length field in the MOBI 6 header correctly. "
+      tickets: [ 1003489 ] 
+
+    - title: "ODT Input: More workarounds for LibreOffice 3.5's habit of inserting pointless margin:100% directives everywhere."
+      tickets: [ 1002702 ] 
+
+    - title: "Fix regression that broke smarten punctuation when quotes were next to html tags."
+      tickets: [ 998900 ] 
+
+    - title: "Fix published date from ozon.ru wrong in some timezones"
+      tickets: [ 975338 ]
+
+    - title: "Catalogs: Handle the use of custom columns with non-ascii names correctly"
+      tickets: [1001437]
+
+    - title: "Conversion pipeline: Remove the attempt to detect and autocorrect if text will go off the left edge of the page, as it was a rather crude heuristic. Also do not remove fake margins if the book uses negative text indents on the margined elements."
+
+    - title: "KF8 Output: Set offsets to tags in the skeleton the same way kindlegen does. Also linearize non linear ToCs to ensure section to section jumping works."
+
+    - title: "Conversion pipeline: Use correct default value of 'inherit' for font-family and font-size when normalizing the shorthand font property."
+
+    - title: "When running python scripts via calibre-debug ensure that user plugins are loaded"
+
+  improved recipes:
+    - Business Week Magazine
+    - Metro Nieuws NL
+
+  new recipes:
+    - title: Attac.es 
+      author: Marc Busque
+
+    - title: Drytooling.com
+      author: Damian Granowski
+
+    - title: Shortlist.com 
+      author: Dave ASbury
+
+    - title: National Geographic (es) 
+      author: vakya
+
 - version: 0.8.52
   date: 2012-05-18
 

manual/conf.py

@@ -14,7 +14,7 @@
 import sys, os
 
 # If your extensions are in another directory, add it here.
-sys.path.append(os.path.abspath('../../../'))
+sys.path.append(os.path.abspath('../src'))
 sys.path.append(os.path.abspath('.'))
 __appname__ = os.environ.get('__appname__', 'calibre')
 __version__ = os.environ.get('__version__', '0.0.0')
@@ -98,7 +98,7 @@ html_favicon = 'favicon.ico'
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the built-in static files,
 # so a file named "default.css" will overwrite the built-in "default.css".
-html_static_path = ['resources', '../../../icons/favicon.ico']
+html_static_path = ['resources', '../icons/favicon.ico']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.

manual/creating_plugins.rst

@@ -172,7 +172,7 @@ You can see the ``prefs`` object being used in main.py:
     :pyobject: DemoDialog.config
 
 
-The different types of plugins
+The plugin API
 --------------------------------
 
 As you may have noticed above, a plugin in |app| is a class. There are different classes for the different types of plugins in |app|.

manual/custom.py

@@ -5,9 +5,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, re, textwrap
 
-sys.path.insert(0, os.path.abspath('../../'))
-sys.extensions_location = '../plugins'
-sys.resources_location  = '../../../resources'
+sys.path.insert(0, os.path.abspath('../src'))
+sys.extensions_location = '../src/calibre/plugins'
+sys.resources_location  = '../resources'
 
 from sphinx.util.console import bold
 
@@ -116,44 +116,42 @@ def generate_ebook_convert_help(preamble, info):
     from calibre.utils.logging import default_log
     preamble = re.sub(r'http.*\.html', ':ref:`conversion`', preamble)
     raw = preamble + textwrap.dedent('''
-    Since the options supported by ebook-convert vary depending on both the
-    input and the output formats, the various combinations are listed below:
+    The options and default values for the options change depending on both the
+    input and output formats, so you should always check with::
+
+        ebook-convert myfile.input_format myfile.output_format -h
+
+    Below are the options that are common to all conversion, followed by the
+    options specific to every input and output format
 
     ''')
-    toc = {}
-    sec_templ = textwrap.dedent('''\
-        .. include:: ../global.rst
-
-        {0}
-        ================================================================
-
-        .. contents:: Contents
-          :depth: 1
-          :local:
-
-    ''')
-    for i, ip in enumerate(input_format_plugins()):
-        sraw = sec_templ.format(ip.name)
-        toc[ip.name] = 'ebook-convert-%d'%i
-        for op in output_format_plugins():
-            title = ip.name + ' to ' + op.name
     parser, plumber = create_option_parser(['ebook-convert',
-                'dummyi.'+list(ip.file_types)[0],
-                'dummyo.'+op.file_type, '-h'], default_log)
-            cmd = 'ebook-convert '+list(ip.file_types)[0]+' '+op.file_type
+        'dummyi.mobi', 'dummyo.epub', '-h'], default_log)
     groups = [(None, None, parser.option_list)]
     for grp in parser.option_groups:
-                groups.append((grp.title, grp.description, grp.option_list))
-            options = '\n'.join(render_options(cmd, groups, False))
-            sraw += title+'\n------------------------------------------------------\n\n'
-            sraw += options + '\n\n'
-        update_cli_doc(os.path.join('cli', toc[ip.name]+'.rst'), sraw, info)
+        if grp.title not in {'INPUT OPTIONS', 'OUTPUT OPTIONS'}:
+            groups.append((grp.title.title(), grp.description, grp.option_list))
+    options = '\n'.join(render_options('ebook-convert', groups, False))
+
+    raw += '\n\n.. contents::\n  :local:'
+
+    raw += '\n\n' + options
+    for pl in sorted(input_format_plugins(), key=lambda x:x.name):
+        parser, plumber = create_option_parser(['ebook-convert',
+            'dummyi.'+list(pl.file_types)[0], 'dummyo.epub', '-h'], default_log)
+        groups = [(pl.name+ ' Options', '', g.option_list) for g in
+                parser.option_groups if g.title == "INPUT OPTIONS"]
+        prog = 'ebook-convert-'+(pl.name.lower().replace(' ', '-'))
+        raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
+    for pl in sorted(output_format_plugins(), key=lambda x: x.name):
+        parser, plumber = create_option_parser(['ebook-convert', 'd.epub',
+            'dummyi.'+pl.file_type, '-h'], default_log)
+        groups = [(pl.name+ ' Options', '', g.option_list) for g in
+                parser.option_groups if g.title == "OUTPUT OPTIONS"]
+        prog = 'ebook-convert-'+(pl.name.lower().replace(' ', '-'))
+        raw += '\n\n' + '\n'.join(render_options(prog, groups, False, True))
 
-    toct = '\n\n.. toctree::\n    :maxdepth: 2\n\n'
-    for ip in sorted(toc):
-        toct += '    ' + toc[ip]+'\n'
 
-    raw += toct+'\n\n'
     update_cli_doc(os.path.join('cli', 'ebook-convert.rst'), raw, info)
 
 def update_cli_doc(path, raw, info):

manual/customize.rst

@@ -39,7 +39,7 @@ Tweaks
 Tweaks are small changes that you can specify to control various aspects of |app|'s behavior. You can change them by going to Preferences->Advanced->Tweaks.
 The default values for the tweaks are reproduced below
 
-.. literalinclude:: ../../../resources/default_tweaks.py
+.. literalinclude:: ../resources/default_tweaks.py
 
 
 Overriding icons, templates, et cetera

manual/develop.rst

@@ -45,6 +45,16 @@ All the |app| python code is in the ``calibre`` package. This package contains t
       The format independent code is all in ebooks.oeb and the format dependent code is in ebooks.format_name.
 
         * Metadata reading, writing, and downloading is all in ebooks.metadata
+        * Conversion happens in a pipeline, for the structure of the pipeline,
+          see :ref:`conversion-introduction`. The pipeline consists of an input
+          plugin, various transforms and an output plugin. The code constructs
+          and drives the pipeline is in plumber.py. The pipeline works on a
+          representation of an ebook that is like an unzipped epub, with
+          manifest, spine, toc, guide, html content, etc. The
+          class that manages this representation is OEBBook in oeb/base.py. The
+          various transformations that are applied to the book during
+          conversions live in `oeb/transforms/*.py`. And the input and output
+          plugins live in `conversion/plugins/*.py`.
 
     * library - The database back-end and the content server. See library.database2 for the interface to the |app| library. library.server is the |app| Content Server.
     * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer.

manual/index.rst

@@ -17,7 +17,7 @@ To get started with more advanced usage, you should read about the :ref:`Graphic
 
 .. only:: online
     
-    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_.
+    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_ and `AZW3 (Kindle Fire) format <calibre.azw3>`_.
     
 Sections
 ------------

manual/template_ref_generate.py

@@ -55,7 +55,7 @@ The python implementation of the template functions is passed in a Metadata obje
 
     The set of standard metadata fields.
 
-.. literalinclude:: ../ebooks/metadata/book/__init__.py
+.. literalinclude:: ../src/calibre/ebooks/metadata/book/__init__.py
    :lines: 7-
 '''
 

recipes/banat_news.recipe

@@ -0,0 +1,68 @@
+
+'''
+www.philstar.com
+'''
+
+import time
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class BanatNews(BasicNewsRecipe):
+    title                  	= 'Banat News'
+    custom_title 	= "Banat News - " + time.strftime('%d %b %Y %I:%M %p')
+    __author__             	= 'jde'
+    __date__                       = '31 May 2012'
+    __version__                   = '1.0'
+    description            	= 'Banat News is a daily Cebuano-language newspaper based in Cebu, Philippines - philstar.com is a Philippine news and entertainment portal for the Filipino global community.   It is the online presence of the STAR Group of Publications, a leading publisher of newspapers and magazines in the Philippines.'
+    language               	= 'ceb'
+    publisher              	= 'The Philippine STAR'
+    category               	= 'news, Philippines'
+    tags 		= 'news, Philippines'
+    cover_url        	= 'http://www.philstar.com/images/logo_Banat.jpg'
+    masthead_url        	= 'http://www.philstar.com/images/logo_Banat.jpg'
+    oldest_article 	= 1.5 #days
+    max_articles_per_feed  	= 25
+    simultaneous_downloads = 10
+    publication_type 	= 'newspaper'
+    timefmt 		= ' [%a, %d %b %Y %I:%M %p]'
+    no_stylesheets         	= True
+    use_embedded_content  = False
+    encoding        	= None
+    recursions      	= 0
+    needs_subscription 	= False
+    remove_javascript 	= True
+    remove_empty_feeds    	= True
+    auto_cleanup 	= False
+
+    remove_tags = [dict(name='img',  attrs={'id':'Image1'})    #Logo
+	       ,dict(name='span', attrs={'id':'ControlArticle1_LabelHeader'})   #Section (Headlines, Nation, Metro, ...)
+	       ,dict(name='a',      attrs={'id':'ControlArticle1_FormView1_hlComments'})   #Comments
+	       ,dict(name='img',  attrs={'src':'images/post-comments.jpg'})   #View Comments
+	       ,dict(name='a',      attrs={'id':'ControlArticle1_FormView1_ControlPhotoAndCaption1_hlImageCaption'})  #Zoom
+	]
+    conversion_options = { 'title'   : custom_title,
+                           'comments'    : description,
+                           'tags'             : tags,
+                           'language'      : language,
+                           'publisher'      : publisher,
+                           'authors'        : publisher,
+                           'smarten_punctuation' : True
+                         }
+
+    feeds = [
+               ('Balita'      		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=101' )
+              ,('Opinyon'       		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=102' )
+              ,('Kalingawan'     		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=104' )
+              ,('Showbiz'          		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=62' )
+              ,('Palaro'            		, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=103' )
+              ,('Imong Kapalaran'        	, 'http://rss.philstar.com/Rss.aspx?publicationSubCategoryId=105' )
+           	 ]
+
+# process the printer friendly version of article
+    def print_version(self, url):
+          return url.replace('/Article', '/ArticlePrinterFriendly')
+
+# obtain title from printer friendly version of article; avoiding  add_toc_thumbnail changing title when article has image
+    def populate_article_metadata(self, article, soup, first):
+          article.title = soup.find('span', {'id': 'ControlArticle1_FormView1_ArticleHeaderLabel'}).contents[0].strip()
+
+

recipes/birmingham_post.recipe

@@ -5,11 +5,11 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
     timefmt = ''
     __author__ = 'Dave Asbury'
     cover_url = 'http://1.bp.blogspot.com/_GwWyq5eGw9M/S9BHPHxW55I/AAAAAAAAB6Q/iGCWl0egGzg/s320/Birmingham+post+Lite+front.JPG'
-    oldest_article = 1
-    max_articles_per_feed = 20
+    oldest_article = 2
+    max_articles_per_feed = 12
     remove_empty_feeds = True
     remove_javascript     = True
-    auto_cleanup = True
+    #auto_cleanup = True
     language = 'en_GB'
 
 
@@ -17,8 +17,11 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
 
 
     keep_only_tags = [
-    #dict(name='h1',attrs={'id' : 'article-headline'}),
-                    #dict(attrs={'class':['article-meta-author','article-meta-date','article main','art-o art-align-center otm-1 ']}),
+    	    dict(name='h1',attrs={'id' : 'article-headline'}),
+                        dict(attrs={'class':['article-meta-author','article-meta-date','article main','art-o art-align-center otm-1 ']}),
+	    dict(name='div',attrs={'class' : 'article-image full'}),
+	    dict(attrs={'clas' : 'art-o art-align-center otm-1 '}),
+    	    dict(name='div',attrs={'class' : 'article main'}),
 #dict(name='p')
     #dict(attrs={'id' : 'three-col'})
         ]
@@ -28,7 +31,7 @@ class AdvancedUserRecipe1306097511(BasicNewsRecipe):
                                ]
     feeds          = [
         #(u'News',u'http://www.birminghampost.net/news/rss.xml'),
-        (u'Local News', u'http://www.birminghampost.net/news/west-midlands-news/rss.xml'),
+        (u'West Mids. News', u'http://www.birminghampost.net/news/west-midlands-news/rss.xml'),
         (u'UK News', u'http://www.birminghampost.net/news/uk-news/rss.xml'),
         (u'Sports',u'http://www.birminghampost.net/midlands-birmingham-sport/rss.xml'),
         (u'Bloggs & Comments',u'http://www.birminghampost.net/comment/rss.xml')

recipes/chinadaily.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010 - 2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.chinadaily.com.cn
 '''
@@ -21,7 +21,11 @@ class Pagina12(BasicNewsRecipe):
     remove_empty_feeds    = True
     publication_type      = 'newsportal'
     masthead_url          = 'http://www.chinadaily.com.cn/15421.files/chinadailylogo_e_20100301.jpg'
-    extra_css             = ' body{font-family: Arial,Helvetica,sans-serif } '
+    extra_css             = """ 
+                                body{font-family: Arial,Helvetica,sans-serif }
+                                .titlebox{font-family: "Times New Roman",Times,serif}
+                                .timebox, .authorbox{font-size: x-small}
+                            """
 
     conversion_options = {
                           'comment'   : description
@@ -31,7 +35,7 @@ class Pagina12(BasicNewsRecipe):
                         }
 
     remove_tags = [dict(name=['object','embed','iframe','table'])]
-    keep_only_tags = [dict(attrs={'id':['Title_e','Content']})]
+    keep_only_tags = [dict(attrs={'class':['titlebox', 'timebox', 'authorbox', 'cont-ins']})]
 
 
     feeds = [
@@ -41,8 +45,3 @@ class Pagina12(BasicNewsRecipe):
              ,(u'Sports'  , u'http://www.chinadaily.com.cn/rss/sports_rss.xml'  )
              ,(u'Opinions', u'http://www.chinadaily.com.cn/rss/opinion_rss.xml' )
             ]
-
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        return soup

recipes/chr_mon.recipe

@@ -1,152 +1,113 @@
-#!/usr/bin/env  python
 __license__   = 'GPL v3'
-__author__      = 'Kovid Goyal and Sujata Raman, Lorenzo Vigentini'
-__copyright__   = '2009, Kovid Goyal and Sujata Raman'
-__version__     = 'v1.02'
-__date__        = '10, January 2010'
-__description__ = 'Providing context and clarity on national and international news, peoples and cultures'
-
-'''csmonitor.com'''
+__copyright__ = '2012, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.csmonitor.com
+'''
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
-
-class ChristianScienceMonitor(BasicNewsRecipe):
-
-    __author__    = 'Kovid Goyal'
-    description   = 'Providing context and clarity on national and international news, peoples and cultures'
-
-    cover_url      = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
-    title          = 'Christian Science Monitor'
+class CSMonitor(BasicNewsRecipe):
+    title                 = 'The Christian Science Monitor - daily'
+    __author__            = 'Darko Miletic'
+    description           = 'The Christian Science Monitor is an international news organization that delivers thoughtful, global coverage via its website, weekly magazine, daily news briefing, and email newsletters.'
     publisher             = 'The Christian Science Monitor'
-    category       = 'News, politics, culture, economy, general interest'
-
-    language = 'en'
-    encoding = 'utf-8'
-    timefmt        = '[%a, %d %b, %Y]'
-
-    oldest_article        = 16
-    max_articles_per_feed = 20
-    use_embedded_content  = False
-    recursion             = 10
-
-    remove_javascript     = True
+    category              = 'news, politics, USA'
+    oldest_article        = 2
+    max_articles_per_feed = 200
     no_stylesheets        = True
-    requires_version = (0, 8, 39)
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://www.csmonitor.com/extension/csm_base/design/csm_design/images/csmlogo_179x46.gif'
+    extra_css             = """
+                               body{font-family: Arial,Tahoma,Verdana,Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               .head {font-family: Georgia,"Times New Roman",Times,serif}
+                               .sByline,.caption{font-size: x-small}
+                               .hide{display: none}
+                               .sLoc{font-weight: bold}
+                               ul{list-style-type: none}
+                            """
 
-    def preprocess_raw_html(self, raw, url):
-        try:
-            from html5lib import parse
-            root = parse(raw, namespaceHTMLElements=False,
-                    treebuilder='lxml').getroot()
-            from lxml import etree
-            for tag in root.xpath(
-                    '//script|//style|//noscript|//meta|//link|//object'):
-                tag.getparent().remove(tag)
-            for elem in list(root.iterdescendants(tag=etree.Comment)):
-                elem.getparent().remove(elem)
-            ans = etree.tostring(root, encoding=unicode)
-            ans = re.sub('.*<html', '<html', ans, flags=re.DOTALL)
-            return ans
-        except:
-            import traceback
-            traceback.print_exc()
-            raise
-
-    def index_to_soup(self, url):
-        raw = BasicNewsRecipe.index_to_soup(self, url,
-                raw=True).decode('utf-8')
-        raw = self.preprocess_raw_html(raw, url)
-        return BasicNewsRecipe.index_to_soup(self, raw)
-
-    def append_page(self, soup, appendtag, position):
-        nav = soup.find('div',attrs={'class':'navigation'})
-        if nav:
-            pager = nav.findAll('a')
-            for part in pager:
-                if 'Next' in part:
-                    nexturl = ('http://www.csmonitor.com' +
-                           re.findall(r'href="(.*?)"', str(part))[0])
-                    soup2 = self.index_to_soup(nexturl)
-                    texttag = soup2.find('div',
-                                 attrs={'class': re.compile('list-article-.*')})
-                    trash_c = soup2.findAll(attrs={'class': 'list-description'})
-                    trash_h = soup2.h1
-                    for tc in trash_c: tc.extract()
-                    trash_h.extract()
-
-                    newpos = len(texttag.contents)
-                    self.append_page(soup2, texttag, newpos)
-                    texttag.extract()
-                    appendtag.insert(position, texttag)
-
-    def preprocess_html(self, soup):
-        PRINT_RE = re.compile(r'/layout/set/print/content/view/print/[0-9]*')
-        html = str(soup)
-        try:
-            print_found = PRINT_RE.findall(html)
-        except Exception:
-            pass
-        if print_found:
-            print_url = 'http://www.csmonitor.com' + print_found[0]
-            print_soup = self.index_to_soup(print_url)
-        else:
-            self.append_page(soup, soup.body, 3)
-
-            trash_a = soup.findAll(attrs={'class': re.compile('navigation.*')})
-            trash_b = soup.findAll(attrs={'style': re.compile('.*')})
-            trash_d = soup.findAll(attrs={'class': 'sByline'})
-            for ta in trash_a: ta.extract()
-            for tb in trash_b: tb.extract()
-            for td in trash_d: td.extract()
-
-            print_soup = soup
-        return print_soup
-
-    extra_css      = '''
-                        h1{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: large}
-                        .sub{ color:#000000;font-family: Georgia,Times,"Times New Roman",serif; font-size: small;}
-                        .byline{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
-                        .postdate{color:#999999 ;  font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
-                        h3{color:#999999 ;  font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
-                        .photoCutline{ color:#333333 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
-                        .photoCredit{ color:#999999 ; font-family:Arial,Helvetica,sans-serif ; font-size: x-small; }
-                        #story{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
-                        #main{font-family:Arial,Tahoma,Verdana,Helvetica,sans-serif ; font-size: small; }
-                        #photo-details{ font-family:Arial,Helvetica,sans-serif ; color:#999999; font-size: x-small;}
-                        span.name{color:#205B87;font-family: Georgia,Times,"Times New Roman",serif; font-size: x-small}
-                        p#dateline{color:#444444 ;  font-family:Arial,Helvetica,sans-serif ; font-style:italic;} '''
-
-    feeds          = [(u'Top Stories', u'http://rss.csmonitor.com/feeds/top'),
-                        (u'World' , u'http://rss.csmonitor.com/feeds/world'),
-                        (u'USA' , u'http://rss.csmonitor.com/feeds/usa'),
-                        (u'Commentary' , u'http://rss.csmonitor.com/feeds/commentary'),
-                        (u'Money' , u'http://rss.csmonitor.com/feeds/wam'),
-                        (u'Learning' , u'http://rss.csmonitor.com/feeds/learning'),
-                        (u'Living', u'http://rss.csmonitor.com/feeds/living'),
-                        (u'Innovation', u'http://rss.csmonitor.com/feeds/scitech'),
-                        (u'Gardening', u'http://rss.csmonitor.com/feeds/gardening'),
-                        (u'Environment',u'http://rss.csmonitor.com/feeds/environment'),
-                        (u'Arts', u'http://rss.csmonitor.com/feeds/arts'),
-                        (u'Books', u'http://rss.csmonitor.com/feeds/books'),
-                        (u'Home Forum' , u'http://rss.csmonitor.com/feeds/homeforum')
-                     ]
-
-    keep_only_tags = [dict(name='div', attrs={'id':'mainColumn'}), ]
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
 
     remove_tags = [
-                        dict(name='div', attrs={'id':['story-tools','videoPlayer','storyRelatedBottom','enlarge-photo','photo-paginate']}),
-                        dict(name=['div','a'], attrs={'class':
-                            ['storyToolbar cfx','podStoryRel','spacer3',
-                                'divvy spacer7','comment','storyIncludeBottom',
-                                'hide', 'podBrdr']}),
-                        dict(name='ul', attrs={'class':[ 'centerliststories']}) ,
-                        dict(name='form', attrs={'id':[ 'commentform']}) ,
-          dict(name='div', attrs={'class': ['ui-comments']})
+                     dict(name=['meta','link','iframe','object','embed'])
+                    ,dict(attrs={'class':re.compile('(^|| )podStoryRel($|| )', re.DOTALL)})
+                    ,dict(attrs={'class':['bottom-rel','hide']})
+                    ,dict(attrs={'id':['pgallerycarousel_enlarge','pgallerycarousel_related']})
+                  ]
+    keep_only_tags = [
+                        dict(name='h1', attrs={'class':'head'})
+                       ,dict(name='h2', attrs={'class':'subhead'})
+                       ,dict(attrs={'class':['sByline','thePhoto','ui-body-header']})
+                       ,dict(attrs={'class':re.compile('(^|| )sBody($|| )', re.DOTALL)})
+                     ]
+    remove_attributes=['xmlns:fb']
+
+    feeds = [
+              (u'USA'           , u'http://rss.csmonitor.com/feeds/usa'        )
+             ,(u'World'         , u'http://rss.csmonitor.com/feeds/world'      )
+             ,(u'Politics'      , u'http://rss.csmonitor.com/feeds/politics'   )
+             ,(u'Business'      , u'http://rss.csmonitor.com/feeds/wam'        )
+             ,(u'Commentary'    , u'http://rss.csmonitor.com/feeds/commentary' )
+             ,(u'Books'         , u'http://rss.csmonitor.com/feeds/books'      )
+             ,(u'Arts'          , u'http://rss.csmonitor.com/feeds/arts'       )
+             ,(u'Environment'   , u'http://rss.csmonitor.com/feeds/environment')
+             ,(u'Innovation'    , u'http://rss.csmonitor.com/feeds/scitech'    )
+             ,(u'Living'        , u'http://rss.csmonitor.com/feeds/living'     )
+             ,(u'Science'       , u'http://rss.csmonitor.com/feeds/science'    )
+             ,(u'The Culture'   , u'http://rss.csmonitor.com/feeds/theculture' )
+             ,(u'The Home Forum', u'http://rss.csmonitor.com/feeds/homeforum'  )
+             ,(u'Articles'      , u'http://rss.csmonitor.com/feeds/csarticles' )
             ]
 
-    remove_tags_after = [ dict(name='div', attrs={'class':[ 'ad csmAd']}),
-              dict(name='div', attrs={'class': [re.compile('navigation.*')]}),
-              dict(name='div', attrs={'style': [re.compile('.*')]})
-                        ]
+    def append_page(self, soup):
+        pager = soup.find('div', attrs={'class':'navigation'})
+        if pager:
+           nexttag = pager.find(attrs={'id':'next-button'})
+           if nexttag:
+              nurl = 'http://www.csmonitor.com' + nexttag['href']
+              soup2 = self.index_to_soup(nurl)
+              texttag = soup2.find(attrs={'class':re.compile('(^|| )sBody($|| )', re.DOTALL)})
+              if texttag:
+                  appendtag = soup.find(attrs={'class':re.compile('(^|| )sBody($|| )', re.DOTALL)})
+                  for citem in texttag.findAll(attrs={'class':[re.compile('(^|| )podStoryRel($|| )', re.DOTALL),'bottom-rel','hide']}):
+                      citem.extract()
+                  self.append_page(soup2)
+                  texttag.extract()
+                  pager.extract()
+                  appendtag.append(texttag)
+
+    def preprocess_html(self, soup):
+        self.append_page(soup)
+        pager = soup.find('div', attrs={'class':'navigation'})
+        if pager:
+           pager.extract()
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if 'scorecardresearch' in item['src']:
+               item.extract()
+            else:
+                if not item.has_key('alt'):
+                   item['alt'] = 'image'
+        return soup