Sync to trunk.

2025-07-09 03:04:10 -04:00 · 2010-12-13 18:25:42 -05:00 · 2010-12-13 18:25:42 -05:00 · f62b520416
commit f62b520416
parent 4dd794083f 3a1fd7af56
127 changed files with 236216 additions and 123728 deletions
--- a/Changelog.yaml
+++ b/Changelog.yaml
@ -4,6 +4,81 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.

+- version: 0.7.33
+  date: 2010-12-10
+
+  new features:
+    - title: "Language sensitive sorting"
+      type: major
+      description: >
+        "calibre now sorts using language specific rules. The language used is the language of the calibre interface, which can be changed via Preferences->Look & Feel. There is also
+        a tweak that allows you to use a different language from the one used for the calibre interface. Powered by the ICU library."
+
+    - title: "Add an action to merge only formats and leave metadata alone (Shift+Alt+M)"
+      tickets: [7709]
+
+    - title: "Add a tweak to control which custom columns are displayed in the Book details panel."
+
+    - title: "Implement a more sophisticated 'functional programming' template language. See the User Manual for details."
+
+    - title: "Speed up deleting of large numbers of books and show progress while doing so"
+
+    - title: "Adding books: Dont refresh the Tag Browser while adding multiple books. Should speed up the adding of large numbers of books."
+
+    - title: "Edit metadata dialog: When trying to download metadata, if there are multiple matches indicate which matches have a cover and summary in the list. Also add an option to automatically download the cover of the selected match."
+
+    - title: "Drivers for the HTC Legend and Samsung Epic"
+
+    - title: "FB2 Output: Convert SVG images in the input document to raster images"
+
+    - title: "News download: Localize the navigation bars in the downloaded news to the language the user has selected for their calibre interface"
+
+  bug fixes:
+    - title: "Various fixes to the Title Case function"
+      tickets: [7846]
+
+    - title: "Content server: Fix --url-prefix being ignored for links at the Top level"
+
+    - title: "News download: When generating periodicals for the SONY use the local timezone in the SONY specific metadata"
+
+    - title: "Fix bug in cover cache that could cause it to keep a large number of covers in memory. Showed up when adding large numbers of books to calibre."
+      tickets: [7813]
+
+    - title: "Adding books: Run in the main thread to prevent unfortunate interactions with the metadata backup. Also fix regression that broke the Abort button."
+
+    - title: "Fix a crash on OS X if OK is clicked inthe edit metadata button while downloading a cover"
+      tickets: [7716]
+
+    - title: "E-book viewer: Fix a regression that prevented booksmarks from working with some EPUB files"
+      tickets: [7812]
+
+    - title: "Save to disk: Refactor to not open a database connection in the worker process. Also fix a bug that could lead to save failures not being reported."
+
+    - title: "Fix regression in 0.7.32 that broke opening formats in the ebook viewer from the edit metadata dialog"
+
+    - title: "FB2 Output: Generate output 100% compliant with the FB2 spec"
+
+    - title: "Fix Saved search dropdown box looses selected search"
+      tickets: [7787]
+
+    - title: "TXT Output: Fix an issue where the br to space conversion was not being handled properly."
+
+  improved recipes:
+    - Le Monde
+    - Ming Pao
+    - New Yorker
+
+  new recipes:
+    - title: "ToyoKeizai News and Nikkei Social News"
+      author: "Hiroshi Miura"
+      
+    - title: "St. Louis Post Dispatch"
+      author: "cisaak"
+
+    - title: "Heise Open and Technology Review"
+      author: "Anton Gillert"
+
+
 - version: 0.7.32
  date: 2010-12-03

--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@ -181,19 +181,25 @@ max_content_server_tags_shown=5
 # content_server_will_display is a list of custom fields to be displayed.
 # content_server_wont_display is a list of custom fields not to be displayed.
 # wont_display has priority over will_display.
-# The special value '*' means all custom fields.
+# The special value '*' means all custom fields. The value [] means no entries.
 # Defaults:
 #    content_server_will_display = ['*']
-#    content_server_wont_display = ['']
+#    content_server_wont_display = []
 # Examples:
 # To display only the custom fields #mytags and #genre:
 #   content_server_will_display = ['#mytags', '#genre']
-#   content_server_wont_display = ['']
+#   content_server_wont_display = []
 # To display all fields except #mycomments:
 #   content_server_will_display = ['*']
 #   content_server_wont_display['#mycomments']
 content_server_will_display = ['*']
-content_server_wont_display = ['']
+content_server_wont_display = []
+
+# Same as above (content server) but for the book details pane. Same syntax.
+# As above, this tweak affects only display of custom fields. The standard
+# fields are not affected
+book_details_will_display = ['*']
+book_details_wont_display = []


 # Set the maximum number of sort 'levels' that calibre will use to resort the
--- a/resources/recipes/ajiajin.recipe
+++ b/resources/recipes/ajiajin.recipe
@ -0,0 +1,23 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+ajiajin.com/blog
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AjiajinBlog(BasicNewsRecipe):
+    title          = u'Ajiajin blog'
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 5
+    publication_type = 'blog'
+    max_articles_per_feed = 100
+    description    = 'The next generation internet trends in Japan and Asia'
+    publisher      = ''
+    category       = 'internet, asia, japan'
+    language       = 'en'
+    encoding      = 'utf-8'
+
+    feeds          = [(u'blog', u'http://feeds.feedburner.com/Asiajin')]
+
+
--- a/resources/recipes/chouchoublog.recipe
+++ b/resources/recipes/chouchoublog.recipe
@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+http://ameblo.jp/
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class SakuraBlog(BasicNewsRecipe):
+    title          = u'chou chou blog'
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 4
+    publication_type = 'blog'
+    max_articles_per_feed = 20
+    description    = 'Japanese popular dog blog'
+    publisher      = ''
+    category       = 'dog, pet, japan'
+    language       = 'ja'
+    encoding      = 'utf-8'
+    use_embedded_content = True
+
+    feeds          = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/chouchou1218/rss20.xml')]
+
+    def parse_feeds(self):
+        feeds = BasicNewsRecipe.parse_feeds(self)
+        for curfeed in feeds:
+            delList = []
+            for a,curarticle in enumerate(curfeed.articles):
+                if re.search(r'rssad.jp', curarticle.url):
+                    delList.append(curarticle)
+            if len(delList)>0:
+                for d in delList:
+                    index = curfeed.articles.index(d)
+                    curfeed.articles[index:index+1] = []
+        return feeds
+
--- a/resources/recipes/dilbert.recipe
+++ b/resources/recipes/dilbert.recipe
@ -3,15 +3,16 @@ __copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 http://www.dilbert.com
 '''
-import re

 from calibre.web.feeds.recipes import BasicNewsRecipe
+import re

-class DosisDiarias(BasicNewsRecipe):
+class DilbertBig(BasicNewsRecipe):
    title                  = 'Dilbert'
-    __author__             = 'Darko Miletic'
+    __author__             = 'Darko Miletic and Starson17'
    description            = 'Dilbert'
-    oldest_article         = 5
+    reverse_article_order = True
+    oldest_article         = 15
    max_articles_per_feed  = 100
    no_stylesheets         = True
    use_embedded_content   = True
@ -29,20 +30,23 @@ class DosisDiarias(BasicNewsRecipe):

    feeds = [(u'Dilbert', u'http://feeds.dilbert.com/DilbertDailyStrip' )]

-    preprocess_regexps = [
-                    (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE),
-                        lambda match: 'strip.zoom.gif')
-                            ]
-
-
    def get_article_url(self, article):
        return article.get('feedburner_origlink', None)

+    preprocess_regexps = [
+        (re.compile('strip\..*\.gif', re.DOTALL|re.IGNORECASE), lambda match: 'strip.zoom.gif')
+        ]
+
    def preprocess_html(self, soup):
       for tag in soup.findAll(name='a'):
           if tag['href'].find('http://feedads') >= 0:
              tag.extract()
       return soup

-
-
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {max-width:100%; min-width:100%;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+		'''
--- a/resources/recipes/heise_open.recipe
+++ b/resources/recipes/heise_open.recipe
@ -0,0 +1,38 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Anton Gillert <atx at binaryninja.de>'
+
+'''
+Fetch Heise Open.
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class HeiseOpenDe(BasicNewsRecipe):
+
+    title = 'Heise Open'
+    description = 'Opensource news from Germany'
+    __author__ = 'Anton Gillert'
+    use_embedded_content   = False
+    language = 'de'
+    timefmt = ' [%d %b %Y]'
+    max_articles_per_feed = 40
+    no_stylesheets = True
+
+    feeds =  [ ('Heise Open', 'http://www.heise.de/open/news/news-atom.xml') ]
+
+    def print_version(self, url):
+        return url + '?view=print'
+
+    remove_tags = [dict(id='navi_top'),
+		   dict(id='navi_bottom'),
+		   dict(name='div', attrs={'class':'navi_top_logo'}),
+		   dict(name='img', attrs={'src':'/open/icons/open_logo_2009_weiss.gif'}),
+		   dict(name='h5', attrs={'style':'margin: 0.5em 0;'}),
+		   dict(name='p', attrs={'class':'news_datum'}),
+		   dict(name='p', attrs={'class':'size80'})]
+    remove_tags_after = [dict(name='p', attrs={'class':'size80'})]
+
+    def get_cover_url(self):
+        return 'http://www.heise.de/open/icons/open_logo_2009_weiss.gif'
+
+
--- a/resources/recipes/kahokushinpo.recipe
+++ b/resources/recipes/kahokushinpo.recipe
@ -0,0 +1,31 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+www.kahoku.co.jp
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class KahokuShinpoNews(BasicNewsRecipe):
+    title          = u'\u6cb3\u5317\u65b0\u5831'
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 2
+    max_articles_per_feed = 20
+    description    = 'Tohoku regional news paper in Japan'
+    publisher      = 'Kahoku Shinpo Sha'
+    category       = 'news, japan'
+    language       = 'ja'
+    encoding      = 'Shift_JIS'
+    no_stylesheets = True
+
+    feeds          = [(u'news', u'http://www.kahoku.co.jp/rss/index_thk.xml')]
+
+    keep_only_tags = [ dict(id="page_title"),
+                                   dict(id="news_detail"),
+                                   dict(id="bt_title"),
+                                   {'class':"photoLeft"},
+                                   dict(id="bt_body")
+                                 ]
+    remove_tags = [ {'class':"button"}]
+
--- a/resources/recipes/kompiutierra.recipe
+++ b/resources/recipes/kompiutierra.recipe
@ -0,0 +1,36 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Vadim Dyadkin, dyadkin@gmail.com'
+__author__ = 'Vadim Dyadkin'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Computerra(BasicNewsRecipe):
+    title = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430'
+    recursion = 50
+    oldest_article = 100
+    __author__  = 'Vadim Dyadkin'
+    max_articles_per_feed = 100
+    use_embedded_content  = False
+    simultaneous_downloads = 5
+    language = 'ru'
+    description = u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u044b, \u043e\u043a\u043e\u043b\u043e\u043d\u0430\u0443\u0447\u043d\u044b\u0435 \u0438 \u043e\u043a\u043e\u043b\u043e\u0444\u0438\u043b\u043e\u0441\u043e\u0444\u0441\u043a\u0438\u0435 \u0441\u0442\u0430\u0442\u044c\u0438, \u0433\u0430\u0434\u0436\u0435\u0442\u044b.'
+
+    keep_only_tags = [dict(name='div', attrs={'id': 'content'}),]
+
+
+    feeds = [(u'\u041a\u043e\u043c\u043f\u044c\u044e\u0442\u0435\u0440\u0440\u0430', 'http://feeds.feedburner.com/ct_news/'),]
+
+    remove_tags = [dict(name='div', attrs={'id': ['fin', 'idc-container', 'idc-noscript',]}),
+                   dict(name='ul', attrs={'class': "related_post"}),
+                   dict(name='p', attrs={'class': 'info'}),
+                   dict(name='a', attrs={'rel': 'tag', 'class': 'twitter-share-button', 'type': 'button_count'}),
+                   dict(name='h2', attrs={}),]
+
+    extra_css = 'body { text-align: justify; }'
+
+    def get_article_url(self, article):
+        return article.get('feedburner:origLink', article.get('guid'))
+
--- a/resources/recipes/le_monde.recipe
+++ b/resources/recipes/le_monde.recipe
@ -1,106 +1,89 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Mathieu Godlewski <mathieu at godlewski.fr>'
-'''
-lemonde.fr
-'''
-
 import re
-from calibre.web.feeds.news import BasicNewsRecipe
-
+from calibre.web.feeds.recipes import BasicNewsRecipe

 class LeMonde(BasicNewsRecipe):
-    title          = 'LeMonde.fr'
-    __author__ = 'Mathieu Godlewski and Sujata Raman'
-    description = 'Global news in french'
-    oldest_article = 3
-    language = 'fr'
+    title                  = 'Le Monde'
+    __author__             = 'veezh'
+    description            = 'Actualités'
+    oldest_article         = 1
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    #delay                  = 1
+    use_embedded_content   = False
+    encoding               = 'cp1252'
+    publisher              = 'lemonde.fr'
+    language               = 'fr'
+    conversion_options = {
+                             'comments'        : description
+                            ,'language'        : language
+                            ,'publisher'       : publisher
+                            ,'linearize_tables': True
+                         }

-    max_articles_per_feed = 30
-    no_stylesheets = True
-    remove_javascript = True
+    remove_empty_feeds = True

+    filterDuplicates = True

-   # cover_url='http://abonnes.lemonde.fr/titresdumonde/'+date.today().strftime("%y%m%d")+'/1.jpg'
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup

-
-    extra_css = '''
-                    .dateline{color:#666666;font-family:verdana,sans-serif;font-size:x-small;}
-                    .author{font-family:verdana,sans-serif;font-size:x-small;color:#222222;}
-                    .articleImage{color:#666666;font-family:verdana,sans-serif;font-size:x-small;}
-                    .mainText{font-family:Georgia,serif;color:#222222;}
-                    .LM_articleText{font-family:Arial,Helvetica,sans-serif;}
-                    .LM_titleZone{font-family:Arial,Helvetica,sans-serif;}
-                    .mainContent{font-family:Georgia,serif;}
-                    .LM_content{font-family:Georgia,serif;}
-                    .LM_caption{font-family:Georgia,serif;font-size:-small;}
-                    .LM_imageSource{font-family:Arial,Helvetica,sans-serif;font-size:x-small;color:#666666;}
-                    h1{font-family:Arial,Helvetica,sans-serif;font-size:medium;color:#000000;}
-                    .post{font-family:Arial,Helvetica,sans-serif;}
-                    .mainTitle{font-family:Georgia,serif;}
-                    .content{font-family:Georgia,serif;}
-                    .entry{font-family:Georgia,serif;}
-                    h2{font-family:Arial,Helvetica,sans-serif;font-size:large;}
-                    small{font-family:Arial,Helvetica,sans-serif;  color:#ED1B23;}
-                '''
-
-    feeds =  [
-             ('A la Une', 'http://www.lemonde.fr/rss/une.xml'),
-             ('International', 'http://www.lemonde.fr/rss/sequence/0,2-3210,1-0,0.xml'),
-             ('Europe', 'http://www.lemonde.fr/rss/sequence/0,2-3214,1-0,0.xml'),
-             ('Societe', 'http://www.lemonde.fr/rss/sequence/0,2-3224,1-0,0.xml'),
-             ('Economie', 'http://www.lemonde.fr/rss/sequence/0,2-3234,1-0,0.xml'),
-             ('Medias', 'http://www.lemonde.fr/rss/sequence/0,2-3236,1-0,0.xml'),
-             ('Rendez-vous', 'http://www.lemonde.fr/rss/sequence/0,2-3238,1-0,0.xml'),
-             ('Sports', 'http://www.lemonde.fr/rss/sequence/0,2-3242,1-0,0.xml'),
-             ('Planete', 'http://www.lemonde.fr/rss/sequence/0,2-3244,1-0,0.xml'),
-             ('Culture', 'http://www.lemonde.fr/rss/sequence/0,2-3246,1-0,0.xml'),
-             ('Technologies', 'http://www.lemonde.fr/rss/sequence/0,2-651865,1-0,0.xml'),
-             ('Cinema', 'http://www.lemonde.fr/rss/sequence/0,2-3476,1-0,0.xml'),
-             ('Voyages', 'http://www.lemonde.fr/rss/sequence/0,2-3546,1-0,0.xml'),
-             ('Livres', 'http://www.lemonde.fr/rss/sequence/0,2-3260,1-0,0.xml'),
-             ('Examens', 'http://www.lemonde.fr/rss/sequence/0,2-3404,1-0,0.xml'),
-             ('Opinions', 'http://www.lemonde.fr/rss/sequence/0,2-3232,1-0,0.xml')
-             ]
-    keep_only_tags = [dict(name='div', attrs={'id':["mainTitle","mainContent","LM_content","content"]}),
-                      dict(name='div', attrs={'class':["post"]})
-                      ]
-
-    remove_tags    = [dict(name='img', attrs={'src':'http://medias.lemonde.fr/mmpub/img/lgo/lemondefr_pet.gif'}),
-                                    dict(name='div', attrs={'id':'xiti-logo-noscript'}),
-                                    dict(name='br', attrs={}),
-                                    dict(name='iframe', attrs={}),
-                     dict(name='table', attrs={'id':["toolBox"]}),
-                      dict(name='table', attrs={'class':["bottomToolBox"]}),
-                      dict(name='div', attrs={'class':["pageNavigation","LM_pagination","fenetreBoxesContainer","breakingNews","LM_toolsBottom","LM_comments","LM_tools","pave_meme_sujet_hidden","boxMemeSujet"]}),
-                      dict(name='div', attrs={'id':["miniUne","LM_sideBar"]}),
-    ]
-
-    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE|re.DOTALL), i[1]) for i in
-        [
-            (r'<html.*(<div class="post".*?>.*?</div>.*?<div class="entry">.*?</div>).*You can start editing here.*</html>', lambda match : '<html><body>'+match.group(1)+'</body></html>'),
-            (r'<p>&nbsp;</p>', lambda match : ''),
-            (r'<img src="http://medias\.lemonde\.fr/mmpub/img/let/(.)\.gif"[^>]*><div class=ar-txt>', lambda match : '<div class=ar-txt>'+match.group(1).upper()),
-            (r'<img src="http://medias\.lemonde\.fr/mmpub/img/let/q(.)\.gif"[^>]*><div class=ar-txt>', lambda match : '<div class=ar-txt>"'+match.group(1).upper()),
-            (r'(<div class=desc><b>.*</b></div>).*</body>', lambda match : match.group(1)),
+    preprocess_regexps = [
+        (re.compile(r' \''), lambda match: ' &lsquo;'),
+        (re.compile(r'\''), lambda match: '&rsquo;'),
+        (re.compile(r'"<'), lambda match: '&nbsp;&raquo;<'),
+        (re.compile(r'>"'), lambda match: '>&laquo;&nbsp;'),
+        (re.compile(r'&rsquo;"'), lambda match: '&rsquo;«&nbsp;'),
+        (re.compile(r' "'), lambda match: ' &laquo;&nbsp;'),
+        (re.compile(r'" '), lambda match: '&nbsp;&raquo; '),
+        (re.compile(r'\("'), lambda match: '(&laquo;&nbsp;'),
+        (re.compile(r'"\)'), lambda match: '&nbsp;&raquo;)'),
+        (re.compile(r'"\.'), lambda match: '&nbsp;&raquo;.'),
+        (re.compile(r'",'), lambda match: '&nbsp;&raquo;,'),
+        (re.compile(r'"\?'), lambda match: '&nbsp;&raquo;?'),
+        (re.compile(r'":'), lambda match: '&nbsp;&raquo;:'),
+        (re.compile(r'";'), lambda match: '&nbsp;&raquo;;'),
+        (re.compile(r'"\!'), lambda match: '&nbsp;&raquo;!'),
+        (re.compile(r' :'), lambda match: '&nbsp;:'),
+        (re.compile(r' ;'), lambda match: '&nbsp;;'),
+        (re.compile(r' \?'), lambda match: '&nbsp;?'),
+        (re.compile(r' \!'), lambda match: '&nbsp;!'),
+        (re.compile(r'\s»'), lambda match: '&nbsp;»'),
+        (re.compile(r'«\s'), lambda match: '«&nbsp;'),
+        (re.compile(r' %'), lambda match: '&nbsp;%'),
+        (re.compile(r'\.jpg&nbsp;&raquo; border='), lambda match: '.jpg'),
+        (re.compile(r'\.png&nbsp;&raquo; border='), lambda match: '.png'),
        ]
-    ]

-    article_match_regexps = [ (re.compile(i)) for i in
-        [
-            (r'http://www\.lemonde\.fr/\S+/article/.*'),
-            (r'http://www\.lemonde\.fr/\S+/portfolio/.*'),
-            (r'http://www\.lemonde\.fr/\S+/article_interactif/.*'),
-            (r'http://\S+\.blog\.lemonde\.fr/.*'),
-        ]
-    ]
+    keep_only_tags    = [
+                       dict(name='div', attrs={'class':['contenu']})
+                        ]

-   # def print_version(self, url):
-   #     return re.sub('http://www\.lemonde\.fr/.*_([0-9]+)_[0-9]+\.html.*','http://www.lemonde.fr/web/imprimer_element/0,40-0,50-\\1,0.html' ,url)
+    remove_tags_after = [dict(id='appel_temoignage')]

-    # Used to filter duplicated articles
-    articles_list = []
+    def get_article_url(self, article):
+        link = article.get('link')
+        if 'blog' not in link:
+             return link
+
+
+
+    feeds          = [
+                      ('A la une', 'http://www.lemonde.fr/rss/une.xml'),
+                      ('International', 'http://www.lemonde.fr/rss/tag/international.xml'),
+                      ('Europe', 'http://www.lemonde.fr/rss/tag/europe.xml'),
+                      (u'Société', 'http://www.lemonde.fr/rss/tag/societe.xml'),
+                      ('Economie', 'http://www.lemonde.fr/rss/tag/economie.xml'),
+                      (u'Médias', 'http://www.lemonde.fr/rss/tag/actualite-medias.xml'),
+                      (u'Planète', 'http://www.lemonde.fr/rss/tag/planete.xml'),
+                      ('Culture', 'http://www.lemonde.fr/rss/tag/culture.xml'),
+                      ('Technologies', 'http://www.lemonde.fr/rss/tag/technologies.xml'),
+                      ('Livres', 'http://www.lemonde.fr/rss/tag/livres.xml'),
+
+                    ]

    def get_cover_url(self):
        cover_url = None
@ -111,42 +94,3 @@ class LeMonde(BasicNewsRecipe):
           cover_url = link_item.img['src']

        return cover_url
-
-    def get_article_url(self, article):
-        url=article.get('link',  None)
-        url=url[0:url.find("#")]
-        if url in self.articles_list:
-            self.log_debug(_('Skipping duplicated article: %s')%url)
-            return False
-        if self.is_article_wanted(url):
-            self.articles_list.append(url)
-            if '/portfolio/' in url or '/video/' in url:
-              url = None
-        return url
-        self.log_debug(_('Skipping filtered article: %s')%url)
-        url = article.get('guid', None)
-
-
-        return False
-
-
-    def is_article_wanted(self, url):
-        if self.article_match_regexps:
-            for m in self.article_match_regexps:
-                if m.search(url):
-                    return True
-            return False
-        return False
-
-    def preprocess_html(self, soup):
-
-          for item in soup.findAll(style=True):
-              del item['style']
-
-          for item in soup.findAll(face=True):
-              del item['face']
-          for tag in soup.findAll(name=['ul','li']):
-                tag.name = 'div'
-
-          return soup
-
--- a/resources/recipes/mainichi_it_news.recipe
+++ b/resources/recipes/mainichi_it_news.recipe
@ -1,4 +1,5 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+import re

 class MainichiDailyITNews(BasicNewsRecipe):
    title          = u'\u6bce\u65e5\u65b0\u805e(IT&\u5bb6\u96fb)'
@ -14,6 +15,7 @@ class MainichiDailyITNews(BasicNewsRecipe):

    remove_tags_before = {'class':"NewsTitle"}
    remove_tags = [{'class':"RelatedArticle"}]
+    remove_tags_after = {'class':"Credit"}

    def parse_feeds(self):

@ -29,4 +31,4 @@ class MainichiDailyITNews(BasicNewsRecipe):
                    index = curfeed.articles.index(d)
                    curfeed.articles[index:index+1] = []

-        return feeds   remove_tags_after = {'class':"Credit"}
+        return feeds
--- a/resources/recipes/ming_pao.recipe
+++ b/resources/recipes/ming_pao.recipe
@ -1,8 +1,9 @@
 __license__   = 'GPL v3'
 __copyright__ = '2010, Eddie Lau'
 '''
-modified from Singtao Toronto calibre recipe by rty
 Change Log:
+2010/12/07: add entertainment section, use newspaper front page as ebook cover, suppress date display in section list
+            (to avoid wrong date display in case the user generates the ebook in a time zone different from HKT)
 2010/11/22: add English section, remove eco-news section which is not updated daily, correct
            ordering of articles
 2010/11/12: add news image and eco-news section
@ -17,14 +18,15 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 from contextlib import nested


-from calibre import __appname__, strftime
+from calibre import __appname__
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation
-from calibre.utils.date import now as nowf

 class MPHKRecipe(BasicNewsRecipe):
+    IsKindleUsed = True  # to avoid generating periodical in which CJK characters can't be displayed in section/article view
+
    title          = 'Ming Pao - Hong Kong'
    oldest_article = 1
    max_articles_per_feed = 100
@ -39,13 +41,13 @@ class MPHKRecipe(BasicNewsRecipe):
    encoding = 'Big5-HKSCS'
    recursions = 0
    conversion_options = {'linearize_tables':True}
-    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;}'
-    #extra_css = 'img {float:right; margin:4px;}'
+    timefmt = ''
+    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
    masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
    keep_only_tags = [dict(name='h1'),
-                      #dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page
+                      dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
                      dict(attrs={'class':['photo']}),
-                      dict(attrs={'id':['newscontent']}),
+                      dict(attrs={'id':['newscontent']}), # entertainment page content
                      dict(attrs={'id':['newscontent01','newscontent02']})]
    remove_tags = [dict(name='style'),
                   dict(attrs={'id':['newscontent135']})]  # for the finance page
@ -55,51 +57,68 @@ class MPHKRecipe(BasicNewsRecipe):
                            lambda match: '<h1>'),
                            (re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
                            lambda match: '</h1>'),
+                            (re.compile(r'<p><a href=.+?</a></p>', re.DOTALL|re.IGNORECASE), # for entertainment page
+                            lambda match: '')
                         ]

    def image_url_processor(cls, baseurl, url):
        # trick: break the url at the first occurance of digit, add an additional
        # '_' at the front
        # not working, may need to move this to preprocess_html() method
-        #minIdx = 10000
-        #i0 = url.find('0')
-        #if i0 >= 0 and i0 < minIdx:
-        #   minIdx = i0
-        #i1 = url.find('1')
-        #if i1 >= 0 and i1 < minIdx:
-        #   minIdx = i1
-        #i2 = url.find('2')
-        #if i2 >= 0 and i2 < minIdx:
-        #   minIdx = i2
-        #i3 = url.find('3')
-        #if i3 >= 0 and i0 < minIdx:
-        #   minIdx = i3
-        #i4 = url.find('4')
-        #if i4 >= 0 and i4 < minIdx:
-        #   minIdx = i4
-        #i5 = url.find('5')
-        #if i5 >= 0 and i5 < minIdx:
-        #   minIdx = i5
-        #i6 = url.find('6')
-        #if i6 >= 0 and i6 < minIdx:
-        #   minIdx = i6
-        #i7 = url.find('7')
-        #if i7 >= 0 and i7 < minIdx:
-        #   minIdx = i7
-        #i8 = url.find('8')
-        #if i8 >= 0 and i8 < minIdx:
-        #   minIdx = i8
-        #i9 = url.find('9')
-        #if i9 >= 0 and i9 < minIdx:
-        #   minIdx = i9
-        #return url[0:minIdx] + '_' + url[minIdx+1:]
+#        minIdx = 10000
+#        i0 = url.find('0')
+#        if i0 >= 0 and i0 < minIdx:
+#           minIdx = i0
+#        i1 = url.find('1')
+#        if i1 >= 0 and i1 < minIdx:
+#           minIdx = i1
+#        i2 = url.find('2')
+#        if i2 >= 0 and i2 < minIdx:
+#           minIdx = i2
+#        i3 = url.find('3')
+#        if i3 >= 0 and i0 < minIdx:
+#           minIdx = i3
+#        i4 = url.find('4')
+#        if i4 >= 0 and i4 < minIdx:
+#           minIdx = i4
+#        i5 = url.find('5')
+#        if i5 >= 0 and i5 < minIdx:
+#           minIdx = i5
+#        i6 = url.find('6')
+#        if i6 >= 0 and i6 < minIdx:
+#           minIdx = i6
+#        i7 = url.find('7')
+#        if i7 >= 0 and i7 < minIdx:
+#           minIdx = i7
+#        i8 = url.find('8')
+#        if i8 >= 0 and i8 < minIdx:
+#           minIdx = i8
+#        i9 = url.find('9')
+#        if i9 >= 0 and i9 < minIdx:
+#           minIdx = i9
        return url

-    def get_fetchdate(self):
+    def get_dtlocal(self):
        dt_utc = datetime.datetime.utcnow()
        # convert UTC to local hk time - at around HKT 6.00am, all news are available
        dt_local = dt_utc - datetime.timedelta(-2.0/24)
-        return dt_local.strftime("%Y%m%d")
+        return dt_local
+
+    def get_fetchdate(self):
+        return self.get_dtlocal().strftime("%Y%m%d")
+
+    def get_fetchday(self):
+        # convert UTC to local hk time - at around HKT 6.00am, all news are available
+        return self.get_dtlocal().strftime("%d")
+
+    def get_cover_url(self):
+        cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
+        br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover)
+        except:
+            cover = None
+        return cover

    def parse_index(self):
            feeds = []
@ -127,9 +146,9 @@ class MPHKRecipe(BasicNewsRecipe):
            # if eco_articles:
            #   feeds.append((u'\u74b0\u4fdd Eco News', eco_articles))
            # special - entertainment
-            #ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
-            #if ent_articles:
-            #   feeds.append(('Entertainment', ent_articles))
+            ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
+            if ent_articles:
+                feeds.append((u'\u5f71\u8996 Entertainment', ent_articles))
            return feeds

    def parse_section(self, url):
@ -164,6 +183,7 @@ class MPHKRecipe(BasicNewsRecipe):
        return current_articles

    def parse_eco_section(self, url):
+        dateStr = self.get_fetchdate()
        soup = self.index_to_soup(url)
        divs = soup.findAll(attrs={'class': ['bullet']})
        current_articles = []
@ -173,23 +193,25 @@ class MPHKRecipe(BasicNewsRecipe):
            title = self.tag_to_string(a)
            url = a.get('href', False)
            url = 'http://tssl.mingpao.com/htm/marketing/eco/cfm/' +url
-            if url not in included_urls and url.rfind('Redirect') == -1:
+            if url not in included_urls and url.rfind('Redirect') == -1 and not url.rfind('.txt') == -1 and not url.rfind(dateStr) == -1:
                current_articles.append({'title': title, 'url': url, 'description':''})
                included_urls.append(url)
        return current_articles

-    #def parse_ent_section(self, url):
-    #   dateStr = self.get_fetchdate()
-    #   soup = self.index_to_soup(url)
-    #   a = soup.findAll('a', href=True)
-    #   current_articles = []
-    #   included_urls = []
-    #   for i in a:
-    #       title = self.tag_to_string(i)
-    #       url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
-    #       if url not in included_urls and not url.rfind('.txt') == -1 and not url.rfind(dateStr) == -1 and not title == '':
-    #           current_articles.append({'title': title, 'url': url, 'description': ''})
-    #   return current_articles
+    def parse_ent_section(self, url):
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href=True)
+        a.reverse()
+        current_articles = []
+        included_urls = []
+        for i in a:
+            title = self.tag_to_string(i)
+            url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
+            if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1):
+                current_articles.append({'title': title, 'url': url, 'description': ''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles

    def preprocess_html(self, soup):
        for item in soup.findAll(style=True):
@ -201,21 +223,26 @@ class MPHKRecipe(BasicNewsRecipe):
        return soup

    def create_opf(self, feeds, dir=None):
-        #super(MPHKRecipe,self).create_opf(feeds, dir)
+        if self.IsKindleUsed == False:
+            super(MPHKRecipe,self).create_opf(feeds, dir)
+            return
        if dir is None:
            dir = self.output_dir
        title = self.short_title()
-        if self.output_profile.periodical_date_in_title:
-            title += strftime(self.timefmt)
+        title += ' ' + self.get_fetchdate()
+        #if self.output_profile.periodical_date_in_title:
+        #    title += strftime(self.timefmt)
        mi = MetaInformation(title, [__appname__])
        mi.publisher = __appname__
        mi.author_sort = __appname__
        mi.publication_type = self.publication_type+':'+self.short_title()
-        mi.timestamp = nowf()
+        #mi.timestamp = nowf()
+        mi.timestamp = self.get_dtlocal()
        mi.comments = self.description
        if not isinstance(mi.comments, unicode):
            mi.comments = mi.comments.decode('utf-8', 'replace')
-        mi.pubdate = nowf()
+        #mi.pubdate = nowf()
+        mi.pubdate = self.get_dtlocal()
        opf_path = os.path.join(dir, 'index.opf')
        ncx_path = os.path.join(dir, 'index.ncx')
        opf = OPFCreator(dir, mi)
--- a/resources/recipes/nationalgeographic.recipe
+++ b/resources/recipes/nationalgeographic.recipe
@ -0,0 +1,38 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+nationalgeographic.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class NationalGeographicNews(BasicNewsRecipe):
+    title          = u'National Geographic News'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    remove_javascript = True
+    no_stylesheets = True
+    use_embedded_content = False
+
+    feeds          = [(u'news', u'http://feeds.nationalgeographic.com/ng/News/News_Main')]
+
+    remove_tags_before = dict(id='page_head')
+    remove_tags_after = [dict(id='social_buttons'),{'class':'aside'}]
+    remove_tags = [
+                       {'class':'hidden'}
+
+                     ]
+
+    def parse_feeds(self):
+        feeds = BasicNewsRecipe.parse_feeds(self)
+        for curfeed in feeds:
+            delList = []
+            for a,curarticle in enumerate(curfeed.articles):
+                if re.search(r'ads\.pheedo\.com', curarticle.url):
+                    delList.append(curarticle)
+            if len(delList)>0:
+                for d in delList:
+                    index = curfeed.articles.index(d)
+                    curfeed.articles[index:index+1] = []
+        return feeds
--- a/resources/recipes/nationalgeographicjp.recipe
+++ b/resources/recipes/nationalgeographicjp.recipe
@ -0,0 +1,20 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+nationalgeographic.co.jp
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class NationalGeoJp(BasicNewsRecipe):
+    title          = u'\u30ca\u30b7\u30e7\u30ca\u30eb\u30fb\u30b8\u30aa\u30b0\u30e9\u30d5\u30a3\u30c3\u30af\u30cb\u30e5\u30fc\u30b9'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+
+    feeds          = [(u'news', u'http://www.nationalgeographic.co.jp/news/rss.php')]
+
+    def print_version(self, url):
+        return re.sub(r'news_article.php','news_printer_friendly.php', url)
+
--- a/resources/recipes/nikkei_sub_shakai.recipe
+++ b/resources/recipes/nikkei_sub_shakai.recipe
@ -10,8 +10,8 @@ import mechanize
 from calibre.ptempfile import PersistentTemporaryFile


-class NikkeiNet_sub_life(BasicNewsRecipe):
-    title           = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(\u751f\u6d3b)'
+class NikkeiNet_sub_shakai(BasicNewsRecipe):
+    title           = u'\u65e5\u7d4c\u65b0\u805e\u96fb\u5b50\u7248(Social)'
    __author__      = 'Hiroshi Miura'
    description     = 'News and current market affairs from Japan'
    cover_url       = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
--- a/resources/recipes/paperli_topic.recipe
+++ b/resources/recipes/paperli_topic.recipe
@ -0,0 +1,58 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+paperli
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre import strftime
+
+class paperli_topics(BasicNewsRecipe):
+    # Customize this recipe and change paperli_tag and title below to
+    # download news on your favorite tag
+    paperli_tag = 'climate'
+    title          = u'The #climate Daily - paperli'
+#-------------------------------------------------------------
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    description    = 'paper.li page about '+ paperli_tag
+    publisher      = 'paper.li'
+    category       = 'paper.li'
+    language       = 'en'
+    encoding       = 'utf-8'
+    remove_javascript = True
+    masthead_title = u'The '+ paperli_tag +' Daily'
+    timefmt        = '[%y/%m/%d]'
+    base_url     = 'http://paper.li'
+    index          = base_url+'/tag/'+paperli_tag
+
+
+    def parse_index(self):
+        # get topics
+        topics = []
+        soup   = self.index_to_soup(self.index)
+        topics_lists = soup.find('div',attrs={'class':'paper-nav-bottom'})
+        for item in topics_lists.findAll('li', attrs={'class':""}):
+            itema = item.find('a',href=True)
+            topics.append({'title': itema.string, 'url': itema['href']})
+
+        #get feeds
+        feeds = []
+        for topic in topics:
+            newsarticles = []
+            soup   = self.index_to_soup(''.join([self.base_url, topic['url'] ]))
+            topstories = soup.findAll('div',attrs={'class':'yui-u'})
+            for itt in topstories:
+                itema = itt.find('a',href=True,attrs={'class':'ts'})
+                if itema is not None:
+                    itemd = itt.find('div',text=True, attrs={'class':'text'})
+                    newsarticles.append({
+                                      'title'      :itema.string
+                                     ,'date'     :strftime(self.timefmt)
+                                     ,'url'        :itema['href']
+                                     ,'description':itemd.string
+                                    })
+            feeds.append((topic['title'], newsarticles))
+        return feeds
+
--- a/resources/recipes/the_h.recipe
+++ b/resources/recipes/the_h.recipe
@ -14,7 +14,7 @@ class TheHeiseOnline(BasicNewsRecipe):
    oldest_article = 3
    description    = 'In association with Heise Online'
    publisher      = 'Heise Media UK Ltd.'
-    category       = 'news, technology, security'
+    category       = 'news, technology, security, OSS, internet'
    max_articles_per_feed = 100
    language       = 'en'
    encoding       = 'utf-8'
@ -27,6 +27,12 @@ class TheHeiseOnline(BasicNewsRecipe):
    feeds          = [
                      (u'The H News Feed', u'http://www.h-online.com/news/atom.xml')
                     ]
+    cover_url = 'http://www.h-online.com/icons/logo_theH.gif'
+
+    remove_tags = [
+                        dict(id="logo"),
+                        dict(id="footer")
+                        ]

    def print_version(self, url):
        return url + '?view=print'
--- a/resources/recipes/toyokeizai.recipe
+++ b/resources/recipes/toyokeizai.recipe
@ -0,0 +1,68 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+www.toyokeizai.net
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+import re
+
+class Toyokeizai(BasicNewsRecipe):
+    title          = u'ToyoKeizai News'
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 1
+    max_articles_per_feed = 50
+    description    = 'Japanese traditional economy and business magazine, only for advanced subscribers supported'
+    publisher      = 'Toyokeizai Shinbun Sha'
+    category       = 'economy, magazine, japan'
+    language       = 'ja'
+    encoding       = 'euc-jp'
+    index          = 'http://member.toyokeizai.net/news/'
+    remove_javascript = True
+    no_stylesheets = True
+    masthead_title = u'TOYOKEIZAI'
+    needs_subscription = True
+    timefmt = '[%y/%m/%d]'
+    recursions = 5
+    match_regexps =[ r'page/\d+']
+
+    keep_only_tags = [
+                      dict(name='div', attrs={'class':['news']}),
+                      dict(name='div', attrs={'class':["news_cont"]}),
+                      dict(name='div', attrs={'class':["news_con"]}),
+#                      dict(name='div', attrs={'class':["norightsMessage"]})
+                     ]
+    remove_tags = [{'class':"mt35 mgz"},
+                            {'class':"mt20 newzia"},
+                            {'class':"mt20 fontS"},
+                            {'class':"bk_btn_m"},
+                            dict(id='newzia_connect_member')
+                            ]
+
+    def parse_index(self):
+        feeds = []
+        soup   = self.index_to_soup(self.index)
+        topstories = soup.find('ul',attrs={'class':'list6'})
+        if topstories:
+            newsarticles = []
+            for itt in topstories.findAll('li'):
+                itema = itt.find('a',href=True)
+                itemd = itt.find('span')
+                newsarticles.append({
+                                      'title'      :itema.string
+                                     ,'date'       :re.compile(r"\- ").sub("",itemd.string)
+                                     ,'url'        :'http://member.toyokeizai.net' + itema['href']
+                                     ,'description':itema['title']
+                                    })
+            feeds.append(('news', newsarticles))
+        return feeds
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('http://member.toyokeizai.net/norights/form/')
+            br.select_form(nr=0)
+            br['kaiin_id']   = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
--- a/resources/recipes/tr.recipe
+++ b/resources/recipes/tr.recipe
@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Anton Gillert <atx at binaryninja.de>'
+
+'''
+Fetch Technology Review.
+'''
+from time import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class TechnologyReviewDe(BasicNewsRecipe):
+
+    title = 'Technology Review'
+    description = 'Technology news from Germany'
+    __author__ = 'Anton Gillert'
+    use_embedded_content   = False
+    language = 'de'
+    timefmt = ' [%d %b %Y]'
+    max_articles_per_feed = 40
+    no_stylesheets = True
+
+    feeds =  [ ('Technology Review', 'http://www.heise.de/tr/news-atom.xml') ]
+
+    def print_version(self, url):
+        return url + '?view=print'
+
+    remove_tags = [dict(id='navi_top'),
+		   dict(id='navi_bottom'),
+		   dict(name='div', attrs={'class':'navi_top_logo'}),
+		   dict(name='img', attrs={'src':'/tr/icons/tr_logo2006.gif'}),
+		   dict(name='p', attrs={'class':'size80'})]
+    remove_tags_after = [dict(name='p', attrs={'class':'size80'})]
+
+    def get_cover_url(self):
+        return 'http://www.heise-medien.de/presseinfo/bilder/tr/' + strftime("%y/tr%m%Y.jpg")
+
+
--- a/resources/recipes/uninohimitu.recipe
+++ b/resources/recipes/uninohimitu.recipe
@ -0,0 +1,36 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Hiroshi Miura <miurahr@linux.com>'
+'''
+http://ameblo.jp/sauta19/
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class UniNoHimituKichiBlog(BasicNewsRecipe):
+    title          = u'Uni secret base'
+    __author__     = 'Hiroshi Miura'
+    oldest_article = 2
+    publication_type = 'blog'
+    max_articles_per_feed = 20
+    description    = 'Japanese famous Cat blog'
+    publisher      = ''
+    category       = 'cat, pet, japan'
+    language       = 'ja'
+    encoding      = 'utf-8'
+
+    feeds          = [(u'blog', u'http://feedblog.ameba.jp/rss/ameblo/sauta19/rss20.xml')]
+
+    def parse_feeds(self):
+        feeds = BasicNewsRecipe.parse_feeds(self)
+        for curfeed in feeds:
+            delList = []
+            for a,curarticle in enumerate(curfeed.articles):
+                if re.search(r'rssad.jp', curarticle.url):
+                    delList.append(curarticle)
+            if len(delList)>0:
+                for d in delList:
+                    index = curfeed.articles.index(d)
+                    curfeed.articles[index:index+1] = []
+        return feeds
+
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.32'
+__version__   = '0.7.33'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"

 import re
--- a/src/calibre/devices/cybook/driver.py
+++ b/src/calibre/devices/cybook/driver.py
@ -65,6 +65,7 @@ class ORIZON(CYBOOK):

    BCD         = [0x319]

+    VENDOR_NAME = ['BOOKEEN', 'LINUX']
    WINDOWS_MAIN_MEM = re.compile(r'(CYBOOK_ORIZON__-FD)|(FILE-STOR_GADGET)')
    WINDOWS_CARD_A_MEM = re.compile('(CYBOOK_ORIZON__-SD)|(FILE-STOR_GADGET)')

--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@ -240,7 +240,7 @@ class CollectionsBookList(BookList):
                return 1
            if y is None:
                return -1
-            if isinstance(x, (unicode, str)):
+            if isinstance(x, basestring) and isinstance(y, basestring):
                c = strcmp(force_unicode(x), force_unicode(y))
            else:
                c = cmp(x, y)
--- a/src/calibre/ebooks/conversion/cli.py
+++ b/src/calibre/ebooks/conversion/cli.py
@ -120,7 +120,7 @@ def add_pipeline_options(parser, plumber):
                  [
                      'base_font_size', 'disable_font_rescaling',
                      'font_size_mapping',
-                      'line_height',
+                      'line_height', 'minimum_line_height',
                      'linearize_tables',
                      'extra_css', 'smarten_punctuation',
                      'margin_top', 'margin_left', 'margin_right',
--- a/src/calibre/ebooks/conversion/plumber.py
+++ b/src/calibre/ebooks/conversion/plumber.py
@ -160,13 +160,30 @@ OptionRecommendation(name='disable_font_rescaling',
                   )
        ),

+OptionRecommendation(name='minimum_line_height',
+            recommended_value=120.0, level=OptionRecommendation.LOW,
+            help=_(
+            'The minimum line height, as a percentage of the element\'s '
+            'calculated font size. calibre will ensure that every element '
+            'has a line height of at least this setting, irrespective of '
+            'what the input document specifies. Set to zero to disable. '
+            'Default is 120%. Use this setting in preference to '
+            'the direct line height specification, unless you know what '
+            'you are doing. For example, you can achieve "double spaced" '
+            'text by setting this to 240.'
+            )
+        ),
+

 OptionRecommendation(name='line_height',
            recommended_value=0, level=OptionRecommendation.LOW,
-            help=_('The line height in pts. Controls spacing between consecutive '
-                   'lines of text. By default no line height manipulation is '
-                   'performed.'
-                   )
+            help=_(
+            'The line height in pts. Controls spacing between consecutive '
+            'lines of text. Only applies to elements that do not define '
+            'their own line height. In most cases, the minimum line height '
+            'option is more useful. '
+            'By default no line height manipulation is performed.'
+            )
        ),

 OptionRecommendation(name='linearize_tables',
--- a/src/calibre/ebooks/epub/periodical.py
+++ b/src/calibre/ebooks/epub/periodical.py
@ -6,9 +6,11 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'

 from uuid import uuid4
+import time

 from calibre.constants import __appname__, __version__
 from calibre import strftime, prepare_string_for_xml as xml
+from calibre.utils.date import parse_date

 SONY_METADATA = u'''\
 <?xml version="1.0" encoding="utf-8"?>
@ -87,7 +89,8 @@ def sony_metadata(oeb):
        pass

    try:
-        date = unicode(m.date[0]).split('T')[0]
+        date = parse_date(unicode(m.date[0]),
+                as_utc=False).strftime('%Y-%m-%d')
    except:
        date = strftime('%Y-%m-%d')
    try:
@ -101,7 +104,7 @@ def sony_metadata(oeb):
            publisher=xml(publisher), issue_date=xml(date),
            language=xml(language))

-    updated = strftime('%Y-%m-%dT%H:%M:%SZ')
+    updated = strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())

    def cal_id(x):
        for k, v in x.attrib.items():
--- a/src/calibre/ebooks/html/input.py
+++ b/src/calibre/ebooks/html/input.py
@ -314,6 +314,8 @@ class HTMLInput(InputFormatPlugin):
            rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, \
            xpath
        from calibre import guess_type
+        from calibre.ebooks.oeb.transforms.metadata import \
+            meta_info_to_oeb_metadata
        import cssutils
        self.OEB_STYLES = OEB_STYLES
        oeb = create_oebbook(log, None, opts, self,
@ -321,15 +323,7 @@ class HTMLInput(InputFormatPlugin):
        self.oeb = oeb

        metadata = oeb.metadata
-        if mi.title:
-            metadata.add('title', mi.title)
-        if mi.authors:
-            for a in mi.authors:
-                metadata.add('creator', a, attrib={'role':'aut'})
-        if mi.publisher:
-            metadata.add('publisher', mi.publisher)
-        if mi.isbn:
-            metadata.add('identifier', mi.isbn, attrib={'scheme':'ISBN'})
+        meta_info_to_oeb_metadata(mi, metadata, log)
        if not metadata.language:
            oeb.logger.warn(u'Language not specified')
            metadata.add('language', get_lang().replace('_', '-'))
--- a/src/calibre/ebooks/metadata/html.py
+++ b/src/calibre/ebooks/metadata/html.py
@ -170,7 +170,27 @@ def get_metadata_(src, encoding=None):
        if match:
            series = match.group(1)
    if series:
+        pat = re.compile(r'\[([.0-9]+)\]')
+        match = pat.search(series)
+        series_index = None
+        if match is not None:
+            try:
+                series_index = float(match.group(1))
+            except:
+                pass
+            series = series.replace(match.group(), '').strip()
+
        mi.series = ent_pat.sub(entity_to_unicode, series)
+        if series_index is None:
+            pat = get_meta_regexp_("Seriesnumber")
+            match = pat.search(src)
+            if match:
+                try:
+                    series_index = float(match.group(1))
+                except:
+                    pass
+        if series_index is not None:
+            mi.series_index = series_index

    # RATING
    rating = None
--- a/src/calibre/ebooks/mobi/mobiml.py
+++ b/src/calibre/ebooks/mobi/mobiml.py
@ -184,7 +184,7 @@ class MobiMLizer(object):
                    para.attrib['value'] = str(istates[-2].list_num)
            elif tag in NESTABLE_TAGS and istate.rendered:
                para = wrapper = bstate.nested[-1]
-            elif left > 0 and indent >= 0:
+            elif not self.opts.mobi_ignore_margins and left > 0 and indent >= 0:
                ems = self.profile.mobi_ems_per_blockquote
                para = wrapper = etree.SubElement(parent, XHTML('blockquote'))
                para = wrapper
--- a/src/calibre/ebooks/mobi/output.py
+++ b/src/calibre/ebooks/mobi/output.py
@ -39,6 +39,12 @@ class MOBIOutput(OutputFormatPlugin):
        OptionRecommendation(name='personal_doc', recommended_value='[PDOC]',
            help=_('Tag marking book to be filed with Personal Docs')
        ),
+        OptionRecommendation(name='mobi_ignore_margins',
+            recommended_value=False,
+            help=_('Ignore margins in the input document. If False, then '
+                'the MOBI output plugin will try to convert margins specified'
+                ' in the input document, otherwise it will ignore them.')
+        ),
    ])

    def check_for_periodical(self):
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@ -633,12 +633,12 @@ class Style(object):
            parent = self._getparent()
            if 'line-height' in self._style:
                lineh = self._style['line-height']
+                if lineh == 'normal':
+                    lineh = '1.2'
                try:
-                    float(lineh)
+                    result = float(lineh) * self.fontSize
                except ValueError:
                    result = self._unit_convert(lineh, base=self.fontSize)
-                else:
-                    result = float(lineh) * self.fontSize
            elif parent is not None:
                # TODO: proper inheritance
                result = parent.lineHeight
--- a/src/calibre/ebooks/oeb/transforms/flatcss.py
+++ b/src/calibre/ebooks/oeb/transforms/flatcss.py
@ -245,6 +245,8 @@ class CSSFlattener(object):
            del node.attrib['bgcolor']
        if cssdict.get('font-weight', '').lower() == 'medium':
            cssdict['font-weight'] = 'normal' # ADE chokes on font-weight medium
+
+        fsize = font_size
        if not self.context.disable_font_rescaling:
            _sbase = self.sbase if self.sbase is not None else \
                self.context.source.fbase
@ -258,6 +260,14 @@ class CSSFlattener(object):
                fsize = self.fmap[font_size]
                cssdict['font-size'] = "%0.5fem" % (fsize / psize)
                psize = fsize
+
+        try:
+            minlh = self.context.minimum_line_height / 100.
+            if style['line-height'] < minlh * fsize:
+                cssdict['line-height'] = str(minlh)
+        except:
+            self.oeb.logger.exception('Failed to set minimum line-height')
+
        if cssdict:
            if self.lineh and self.fbase and tag != 'body':
                self.clean_edges(cssdict, style, psize)
@ -290,6 +300,7 @@ class CSSFlattener(object):
            lineh = self.lineh / psize
            cssdict['line-height'] = "%0.5fem" % lineh

+
        if (self.context.remove_paragraph_spacing or
                self.context.insert_blank_line) and tag in ('p', 'div'):
            if item_id != 'calibre_jacket' or self.context.output_profile.name == 'Kindle':
--- a/src/calibre/ebooks/txt/input.py
+++ b/src/calibre/ebooks/txt/input.py
@ -77,10 +77,14 @@ class TXTInput(InputFormatPlugin):
        base = os.getcwdu()
        if hasattr(stream, 'name'):
            base = os.path.dirname(stream.name)
-        htmlfile = open(os.path.join(base, 'temp_calibre_txt_input_to_html.html'),
-                'wb')
-        htmlfile.write(html.encode('utf-8'))
-        htmlfile.close()
+        fname = os.path.join(base, 'index.html')
+        c = 0
+        while os.path.exists(fname):
+            c += 1
+            fname = 'index%d.html'%c
+        htmlfile = open(fname, 'wb')
+        with htmlfile:
+            htmlfile.write(html.encode('utf-8'))
        cwd = os.getcwdu()
        odi = options.debug_pipeline
        options.debug_pipeline = None
--- a/src/calibre/gui2/actions/delete.py
+++ b/src/calibre/gui2/actions/delete.py
@ -5,13 +5,67 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'

-from PyQt4.Qt import QMenu
+from functools import partial
+
+from PyQt4.Qt import QMenu, QObject, QTimer

 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.delete_matching_from_device import DeleteMatchingFromDeviceDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.actions import InterfaceAction

+single_shot = partial(QTimer.singleShot, 10)
+
+class MultiDeleter(QObject):
+
+    def __init__(self, gui, rows, callback):
+        from calibre.gui2.dialogs.progress import ProgressDialog
+        QObject.__init__(self, gui)
+        self.model = gui.library_view.model()
+        self.ids = list(map(self.model.id, rows))
+        self.gui = gui
+        self.failures = []
+        self.deleted_ids = []
+        self.callback = callback
+        single_shot(self.delete_one)
+        self.pd = ProgressDialog(_('Deleting...'), parent=gui,
+                cancelable=False, min=0, max=len(self.ids))
+        self.pd.setModal(True)
+        self.pd.show()
+
+    def delete_one(self):
+        if not self.ids:
+            self.cleanup()
+            return
+        id_ = self.ids.pop()
+        title = 'id:%d'%id_
+        try:
+            title_ = self.model.db.title(id_, index_is_id=True)
+            if title_:
+                title = title_
+            self.model.db.delete_book(id_, notify=False, commit=False)
+            self.deleted_ids.append(id_)
+        except:
+            import traceback
+            self.failures.append((id_, title, traceback.format_exc()))
+        single_shot(self.delete_one)
+        self.pd.value += 1
+        self.pd.set_msg(_('Deleted') + ' ' + title)
+
+    def cleanup(self):
+        self.pd.hide()
+        self.pd = None
+        self.model.db.commit()
+        self.model.db.clean()
+        self.model.books_deleted()
+        self.gui.tags_view.recount()
+        self.callback(self.deleted_ids)
+        if self.failures:
+            msg = ['==> '+x[1]+'\n'+x[2] for x in self.failures]
+            error_dialog(self.gui, _('Failed to delete'),
+                    _('Failed to delete some books, click the Show Details button'
+                    ' for details.'), det_msg='\n\n'.join(msg), show=True)
+
 class DeleteAction(InterfaceAction):

    name = 'Remove Books'
@ -179,8 +233,13 @@ class DeleteAction(InterfaceAction):
            row = None
            if ci.isValid():
                row = ci.row()
-            ids_deleted = view.model().delete_books(rows)
-            self.library_ids_deleted(ids_deleted, row)
+            if len(rows) < 5:
+                ids_deleted = view.model().delete_books(rows)
+                self.library_ids_deleted(ids_deleted, row)
+            else:
+                self.__md = MultiDeleter(self.gui, rows,
+                        partial(self.library_ids_deleted, current_row=row))
+
        else:
            if not confirm('<p>'+_('The selected books will be '
                                   '<b>permanently deleted</b> '
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@ -54,6 +54,10 @@ class EditMetadataAction(InterfaceAction):
        mb.addAction(_('Merge into first selected book - keep others'),
                partial(self.merge_books, safe_merge=True),
                Qt.AltModifier+Qt.Key_M)
+        mb.addSeparator()
+        mb.addAction(_('Merge only formats into first selected book - delete others'),
+                partial(self.merge_books, merge_only_formats=True),
+                Qt.AltModifier+Qt.ShiftModifier+Qt.Key_M)
        self.merge_menu = mb
        self.action_merge.setMenu(mb)
        md.addSeparator()
@ -206,7 +210,7 @@ class EditMetadataAction(InterfaceAction):
            self.gui.library_view.select_rows(ids)

    # Merge books {{{
-    def merge_books(self, safe_merge=False):
+    def merge_books(self, safe_merge=False, merge_only_formats=False):
        '''
        Merge selected books in library.
        '''
@ -220,6 +224,12 @@ class EditMetadataAction(InterfaceAction):
            return error_dialog(self.gui, _('Cannot merge books'),
                        _('At least two books must be selected for merging'),
                        show=True)
+        if len(rows) > 5:
+            if not confirm('<p>'+_('You are about to merge more than 5 books.  '
+                                    'Are you <b>sure</b> you want to proceed?')
+                                +'</p>', 'merge_too_many_books', self.gui):
+                return
+
        dest_id, src_books, src_ids = self.books_to_merge(rows)
        title = self.gui.library_view.model().db.title(dest_id, index_is_id=True)
        if safe_merge:
@ -234,6 +244,22 @@ class EditMetadataAction(InterfaceAction):
                return
            self.add_formats(dest_id, src_books)
            self.merge_metadata(dest_id, src_ids)
+        elif merge_only_formats:
+            if not confirm('<p>'+_(
+                'Book formats from the selected books will be merged '
+                'into the <b>first selected book</b> (%s). '
+                'Metadata in the first selected book will not be changed.'
+                'Author, Title, ISBN and all other metadata will <i>not</i> be merged.<br><br>'
+                'After merger the second and subsequently '
+                'selected books, with any metadata they have will be <b>deleted</b>. <br><br>'
+                'All book formats of the first selected book will be kept '
+                'and any duplicate formats in the second and subsequently selected books '
+                'will be permanently <b>deleted</b> from your calibre library.<br><br>  '
+                'Are you <b>sure</b> you want to proceed?')%title
+            +'</p>', 'merge_only_formats', self.gui):
+                return
+            self.add_formats(dest_id, src_books)
+            self.delete_books_after_merge(src_ids)
        else:
            if not confirm('<p>'+_(
                'Book formats and metadata from the selected books will be merged '
@ -243,15 +269,10 @@ class EditMetadataAction(InterfaceAction):
                'subsequently selected books will be <b>deleted</b>. <br><br>'
                'All book formats of the first selected book will be kept '
                'and any duplicate formats in the second and subsequently selected books '
-                'will be permanently <b>deleted</b> from your computer.<br><br>  '
+                'will be permanently <b>deleted</b> from your calibre library.<br><br>  '
                'Are you <b>sure</b> you want to proceed?')%title
            +'</p>', 'merge_books', self.gui):
                return
-            if len(rows)>5:
-                if not confirm('<p>'+_('You are about to merge more than 5 books.  '
-                                        'Are you <b>sure</b> you want to proceed?')
-                                    +'</p>', 'merge_too_many_books', self.gui):
-                    return
            self.add_formats(dest_id, src_books)
            self.merge_metadata(dest_id, src_ids)
            self.delete_books_after_merge(src_ids)
--- a/src/calibre/gui2/convert/init.py
+++ b/src/calibre/gui2/convert/init.py
@ -10,7 +10,7 @@ import textwrap
 from functools import partial

 from PyQt4.Qt import QWidget, QSpinBox, QDoubleSpinBox, QLineEdit, QTextEdit, \
-    QCheckBox, QComboBox, Qt, QIcon, pyqtSignal
+    QCheckBox, QComboBox, Qt, QIcon, pyqtSignal, QLabel

 from calibre.customize.conversion import OptionRecommendation
 from calibre.ebooks.conversion.config import load_defaults, \
@ -81,6 +81,21 @@ class Widget(QWidget):
        self.apply_recommendations(defaults)
        self.setup_help(get_help)

+        def process_child(child):
+            for g in child.children():
+                if isinstance(g, QLabel):
+                    buddy = g.buddy()
+                    if buddy is not None and hasattr(buddy, '_help'):
+                        g._help = buddy._help
+                        htext = unicode(buddy.toolTip()).strip()
+                        g.setToolTip(htext)
+                        g.setWhatsThis(htext)
+                        g.__class__.enterEvent = lambda obj, event: self.set_help(getattr(obj, '_help', obj.toolTip()))
+                else:
+                    process_child(g)
+        process_child(self)
+
+
    def restore_defaults(self, get_option):
        defaults = GuiRecommendations()
        defaults.merge_recommendations(get_option, OptionRecommendation.LOW,
--- a/src/calibre/gui2/convert/look_and_feel.py
+++ b/src/calibre/gui2/convert/look_and_feel.py
@ -21,7 +21,7 @@ class LookAndFeelWidget(Widget, Ui_Form):
    def __init__(self, parent, get_option, get_help, db=None, book_id=None):
        Widget.__init__(self, parent,
                ['change_justification', 'extra_css', 'base_font_size',
-                    'font_size_mapping', 'line_height',
+                    'font_size_mapping', 'line_height', 'minimum_line_height',
                    'linearize_tables', 'smarten_punctuation',
                    'disable_font_rescaling', 'insert_blank_line',
                    'remove_paragraph_spacing', 'remove_paragraph_spacing_indent_size','input_encoding',
--- a/src/calibre/gui2/convert/look_and_feel.ui
+++ b/src/calibre/gui2/convert/look_and_feel.ui
@ -97,7 +97,7 @@
     </item>
    </layout>
   </item>
-   <item row="3" column="0">
+   <item row="4" column="0">
    <widget class="QLabel" name="label">
     <property name="text">
      <string>Line &amp;height:</string>
@ -107,7 +107,7 @@
     </property>
    </widget>
   </item>
-   <item row="3" column="1" colspan="2">
+   <item row="4" column="1" colspan="2">
    <widget class="QDoubleSpinBox" name="opt_line_height">
     <property name="suffix">
      <string> pt</string>
@ -117,7 +117,7 @@
     </property>
    </widget>
   </item>
-   <item row="4" column="0">
+   <item row="5" column="0">
    <widget class="QLabel" name="label_3">
     <property name="text">
      <string>Input character &amp;encoding:</string>
@ -127,17 +127,17 @@
     </property>
    </widget>
   </item>
-   <item row="4" column="1" colspan="3">
+   <item row="5" column="1" colspan="3">
    <widget class="QLineEdit" name="opt_input_encoding"/>
   </item>
-   <item row="5" column="0" colspan="2">
+   <item row="6" column="0" colspan="2">
    <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
     <property name="text">
      <string>Remove &amp;spacing between paragraphs</string>
     </property>
    </widget>
   </item>
-   <item row="5" column="2" colspan="2">
+   <item row="6" column="2" colspan="2">
    <layout class="QHBoxLayout" name="horizontalLayout_2">
     <item>
      <widget class="QLabel" name="label_4">
@ -164,21 +164,21 @@
     </item>
    </layout>
   </item>
-   <item row="6" column="0">
+   <item row="7" column="0">
    <widget class="QLabel" name="label_5">
     <property name="text">
      <string>Text justification:</string>
     </property>
    </widget>
   </item>
-   <item row="7" column="0">
+   <item row="8" column="0">
    <widget class="QCheckBox" name="opt_linearize_tables">
     <property name="text">
      <string>&amp;Linearize tables</string>
     </property>
    </widget>
   </item>
-   <item row="10" column="0" colspan="4">
+   <item row="11" column="0" colspan="4">
    <widget class="QGroupBox" name="groupBox">
     <property name="title">
      <string>Extra &amp;CSS</string>
@ -190,37 +190,60 @@
     </layout>
    </widget>
   </item>
-   <item row="6" column="2" colspan="2">
+   <item row="7" column="2" colspan="2">
    <widget class="QComboBox" name="opt_change_justification"/>
   </item>
-   <item row="7" column="1" colspan="3">
+   <item row="8" column="1" colspan="3">
    <widget class="QCheckBox" name="opt_asciiize">
     <property name="text">
      <string>&amp;Transliterate unicode characters to ASCII</string>
     </property>
    </widget>
   </item>
-   <item row="8" column="0">
+   <item row="9" column="0">
    <widget class="QCheckBox" name="opt_insert_blank_line">
     <property name="text">
      <string>Insert &amp;blank line</string>
     </property>
    </widget>
   </item>
-   <item row="8" column="1" colspan="2">
+   <item row="9" column="1" colspan="2">
    <widget class="QCheckBox" name="opt_keep_ligatures">
     <property name="text">
      <string>Keep &amp;ligatures</string>
     </property>
    </widget>
   </item>
-   <item row="9" column="0">
+   <item row="10" column="0">
    <widget class="QCheckBox" name="opt_smarten_punctuation">
     <property name="text">
      <string>Smarten &amp;punctuation</string>
     </property>
    </widget>
   </item>
+   <item row="3" column="0">
+    <widget class="QLabel" name="label_6">
+     <property name="text">
+      <string>Minimum &amp;line height:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_minimum_line_height</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1" colspan="2">
+    <widget class="QDoubleSpinBox" name="opt_minimum_line_height">
+     <property name="suffix">
+      <string> %</string>
+     </property>
+     <property name="decimals">
+      <number>1</number>
+     </property>
+     <property name="maximum">
+      <double>900.000000000000000</double>
+     </property>
+    </widget>
+   </item>
  </layout>
 </widget>
 <resources>
--- a/src/calibre/gui2/convert/mobi_output.py
+++ b/src/calibre/gui2/convert/mobi_output.py
@ -25,6 +25,7 @@ class PluginWidget(Widget, Ui_Form):
    def __init__(self, parent, get_option, get_help, db=None, book_id=None):
        Widget.__init__(self, parent,
                ['prefer_author_sort', 'rescale_images', 'toc_title',
+                    'mobi_ignore_margins',
                'dont_compress', 'no_inline_toc', 'masthead_font','personal_doc']
                )
        self.db, self.book_id = db, book_id
--- a/src/calibre/gui2/convert/mobi_output.ui
+++ b/src/calibre/gui2/convert/mobi_output.ui
@ -55,7 +55,7 @@
     </property>
    </widget>
   </item>
-   <item row="5" column="0" colspan="2">
+   <item row="6" column="0" colspan="2">
    <widget class="QGroupBox" name="groupBox">
     <property name="title">
      <string>Kindle options</string>
@ -101,7 +101,7 @@
     </layout>
    </widget>
   </item>
-   <item row="6" column="0">
+   <item row="7" column="0">
    <spacer name="verticalSpacer_2">
     <property name="orientation">
      <enum>Qt::Vertical</enum>
@ -114,6 +114,13 @@
     </property>
    </spacer>
   </item>
+   <item row="5" column="0">
+    <widget class="QCheckBox" name="opt_mobi_ignore_margins">
+     <property name="text">
+      <string>Ignore &amp;margins</string>
+     </property>
+    </widget>
+   </item>
  </layout>
 </widget>
 <resources/>
--- a/src/calibre/gui2/dialogs/comments_dialog.py
+++ b/src/calibre/gui2/dialogs/comments_dialog.py
@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'

-from PyQt4.Qt import Qt, QDialog
+from PyQt4.Qt import Qt, QDialog, QDialogButtonBox
 from calibre.gui2.dialogs.comments_dialog_ui import Ui_CommentsDialog

 class CommentsDialog(QDialog, Ui_CommentsDialog):
@ -20,3 +20,6 @@ class CommentsDialog(QDialog, Ui_CommentsDialog):
        if text is not None:
            self.textbox.setPlainText(text)
        self.textbox.setTabChangesFocus(True)
+        self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK'))
+        self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel'))
+
--- a/src/calibre/gui2/dialogs/comments_dialog.ui
+++ b/src/calibre/gui2/dialogs/comments_dialog.ui
@ -19,15 +19,6 @@
  <property name="windowTitle">
   <string>Edit Comments</string>
  </property>
-  <widget class="QWidget" name="verticalLayoutWidget">
-   <property name="geometry">
-    <rect>
-     <x>10</x>
-     <y>10</y>
-     <width>311</width>
-     <height>211</height>
-    </rect>
-   </property>
   <layout class="QVBoxLayout" name="verticalLayout">
    <item>
     <widget class="QPlainTextEdit" name="textbox"/>
@ -43,7 +34,6 @@
     </widget>
    </item>
   </layout>
-  </widget>
 </widget>
 <resources/>
 <connections>
--- a/src/calibre/gui2/dialogs/confirm_delete.py
+++ b/src/calibre/gui2/dialogs/confirm_delete.py
@ -31,4 +31,5 @@ def confirm(msg, name, parent=None, pixmap='dialog_warning.png'):
    d = Dialog(msg, name, parent)
    d.label.setPixmap(QPixmap(I(pixmap)))
    d.setWindowIcon(QIcon(I(pixmap)))
+    d.resize(d.sizeHint())
    return d.exec_() == d.Accepted
--- a/src/calibre/gui2/dialogs/edit_authors_dialog.py
+++ b/src/calibre/gui2/dialogs/edit_authors_dialog.py
@ -8,13 +8,14 @@ from PyQt4.Qt import Qt, QDialog, QTableWidgetItem, QAbstractItemView
 from calibre.ebooks.metadata import author_to_author_sort
 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.edit_authors_dialog_ui import Ui_EditAuthorsDialog
+from calibre.utils.icu import sort_key, strcmp

 class tableItem(QTableWidgetItem):
    def __ge__(self, other):
-        return unicode(self.text()).lower() >= unicode(other.text()).lower()
+        return sort_key(unicode(self.text())) >= sort_key(unicode(other.text()))

    def __lt__(self, other):
-        return unicode(self.text()).lower() < unicode(other.text()).lower()
+        return sort_key(unicode(self.text())) < sort_key(unicode(other.text()))

 class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):

@ -36,6 +37,7 @@ class EditAuthorsDialog(QDialog, Ui_EditAuthorsDialog):
        self.authors = {}
        auts = db.get_authors_with_ids()
        self.table.setRowCount(len(auts))
+        setattr(self.table, '__lt__', lambda x, y: True if strcmp(x, y) < 0 else False)
        select_item = None
        for row, (id, author, sort) in enumerate(auts):
            author = author.replace('|', ',')
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@ -102,7 +102,7 @@ class MyBlockingBusy(QDialog):
        remove_all, remove, add, au, aus, do_aus, rating, pub, do_series, \
            do_autonumber, do_remove_format, remove_format, do_swap_ta, \
            do_remove_conv, do_auto_author, series, do_series_restart, \
-            series_start_value, do_title_case, clear_series = self.args
+            series_start_value, do_title_case, cover_action, clear_series = self.args


        # first loop: do author and title. These will commit at the end of each
@ -129,6 +129,23 @@ class MyBlockingBusy(QDialog):
                self.db.set_title(id, titlecase(title), notify=False)
            if au:
                self.db.set_authors(id, string_to_authors(au), notify=False)
+            if cover_action == 'remove':
+                self.db.remove_cover(id)
+            elif cover_action == 'generate':
+                from calibre.ebooks import calibre_cover
+                from calibre.ebooks.metadata import fmt_sidx
+                from calibre.gui2 import config
+                mi = self.db.get_metadata(id, index_is_id=True)
+                series_string = None
+                if mi.series:
+                    series_string = _('Book %s of %s')%(
+                        fmt_sidx(mi.series_index,
+                        use_roman=config['use_roman_numerals_for_series_number']),
+                        mi.series)
+
+                cdata = calibre_cover(mi.title, mi.format_field('authors')[-1],
+                        series_string=series_string)
+                self.db.set_cover(id, cdata)
        elif self.current_phase == 2:
            # All of these just affect the DB, so we can tolerate a total rollback
            if do_auto_author:
@ -678,11 +695,16 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
        do_remove_conv = self.remove_conversion_settings.isChecked()
        do_auto_author = self.auto_author_sort.isChecked()
        do_title_case = self.change_title_to_title_case.isChecked()
+        cover_action = None
+        if self.cover_remove.isChecked():
+            cover_action = 'remove'
+        elif self.cover_generate.isChecked():
+            cover_action = 'generate'

        args = (remove_all, remove, add, au, aus, do_aus, rating, pub, do_series,
                do_autonumber, do_remove_format, remove_format, do_swap_ta,
                do_remove_conv, do_auto_author, series, do_series_restart,
-                series_start_value, do_title_case, clear_series)
+                series_start_value, do_title_case, cover_action, clear_series)

        bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.')
                %len(self.ids), args, self.db, self.ids,
--- a/src/calibre/gui2/dialogs/metadata_bulk.ui
+++ b/src/calibre/gui2/dialogs/metadata_bulk.ui
@ -381,7 +381,7 @@ Future conversion of these books will use the default settings.</string>
           </property>
          </widget>
         </item>
-         <item row="15" column="0" colspan="3">
+         <item row="14" column="0" colspan="3">
          <spacer name="verticalSpacer_2">
           <property name="orientation">
            <enum>Qt::Vertical</enum>
@ -394,6 +394,39 @@ Future conversion of these books will use the default settings.</string>
           </property>
          </spacer>
         </item>
+         <item row="13" column="0" colspan="3">
+          <widget class="QGroupBox" name="groupBox">
+           <property name="title">
+            <string>Change &amp;cover</string>
+           </property>
+           <layout class="QVBoxLayout" name="verticalLayout_2">
+            <item>
+             <widget class="QRadioButton" name="cover_no_change">
+              <property name="text">
+               <string>&amp;No change</string>
+              </property>
+              <property name="checked">
+               <bool>true</bool>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QRadioButton" name="cover_remove">
+              <property name="text">
+               <string>&amp;Remove cover</string>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QRadioButton" name="cover_generate">
+              <property name="text">
+               <string>&amp;Generate default cover</string>
+              </property>
+             </widget>
+            </item>
+           </layout>
+          </widget>
+         </item>
        </layout>
       </widget>
       <widget class="QWidget" name="tab">
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@ -240,37 +240,39 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
        self.cover_fetcher = CoverFetcher(None, None, isbn,
                                            self.timeout, title, author)
        self.cover_fetcher.start()
-        self._hangcheck = QTimer(self)
-        self._hangcheck.timeout.connect(self.hangcheck,
-                type=Qt.QueuedConnection)
        self.cf_start_time = time.time()
        self.pi.start(_('Downloading cover...'))
-        self._hangcheck.start(100)
+        QTimer.singleShot(100, self.hangcheck)

    def hangcheck(self):
-        if self.cover_fetcher.is_alive() and \
-            time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT:
+        cf = self.cover_fetcher
+        if cf is None:
+            # Called after dialog closed
+            return
+
+        if cf.is_alive() and \
+            time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT:
+            QTimer.singleShot(100, self.hangcheck)
            return

-        self._hangcheck.stop()
        try:
-            if self.cover_fetcher.is_alive():
+            if cf.is_alive():
                error_dialog(self, _('Cannot fetch cover'),
                    _('<b>Could not fetch cover.</b><br/>')+
                    _('The download timed out.')).exec_()
                return
-            if self.cover_fetcher.needs_isbn:
+            if cf.needs_isbn:
                error_dialog(self, _('Cannot fetch cover'),
                    _('Could not find cover for this book. Try '
                      'specifying the ISBN first.')).exec_()
                return
-            if self.cover_fetcher.exception is not None:
-                err = self.cover_fetcher.exception
+            if cf.exception is not None:
+                err = cf.exception
                error_dialog(self, _('Cannot fetch cover'),
                    _('<b>Could not fetch cover.</b><br/>')+unicode(err)).exec_()
                return
-            if self.cover_fetcher.errors and self.cover_fetcher.cover_data is None:
-                details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in self.cover_fetcher.errors])
+            if cf.errors and cf.cover_data is None:
+                details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in cf.errors])
                error_dialog(self, _('Cannot fetch cover'),
                    _('<b>Could not fetch cover.</b><br/>') +
                    _('For the error message from each cover source, '
@ -278,7 +280,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                return

            pix = QPixmap()
-            pix.loadFromData(self.cover_fetcher.cover_data)
+            pix.loadFromData(cf.cover_data)
            if pix.isNull():
                error_dialog(self, _('Bad cover'),
                             _('The cover is not a valid picture')).exec_()
@ -287,7 +289,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                self.update_cover_tooltip()
                self.cover_changed = True
                self.cpixmap = pix
-                self.cover_data = self.cover_fetcher.cover_data
+                self.cover_data = cf.cover_data
        finally:
            self.fetch_cover_button.setEnabled(True)
            self.unsetCursor()
@ -438,6 +440,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
    def __init__(self, window, row, db, prev=None,
            next_=None):
        ResizableDialog.__init__(self, window)
+        self.cover_fetcher = None
        self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter)
        self.cancel_all = False
        base = unicode(self.author_sort.toolTip())
@ -828,10 +831,6 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
        self.accept()

    def accept(self):
-        cf = getattr(self, 'cover_fetcher', None)
-        if cf is not None and hasattr(cf, 'terminate'):
-            cf.terminate()
-            cf.wait()
        try:
            if self.formats_changed:
                self.sync_formats()
@ -888,14 +887,12 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                        show=True)
            raise
        self.save_state()
+        self.cover_fetcher = None
        QDialog.accept(self)

    def reject(self, *args):
-        cf = getattr(self, 'cover_fetcher', None)
-        if cf is not None and hasattr(cf, 'terminate'):
-            cf.terminate()
-            cf.wait()
        self.save_state()
+        self.cover_fetcher = None
        QDialog.reject(self, *args)

    def read_state(self):
--- a/src/calibre/gui2/dialogs/tag_categories.py
+++ b/src/calibre/gui2/dialogs/tag_categories.py
@ -145,7 +145,7 @@ class TagCategories(QDialog, Ui_TagCategories):
            index = self.all_items[node.data(Qt.UserRole).toPyObject()].index
            if index not in self.applied_items:
                self.applied_items.append(index)
-        self.applied_items.sort(key=lambda x:sort_key(self.all_items[x]))
+        self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name))
        self.display_filtered_categories(None)

    def unapply_tags(self, node=None):
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@ -105,9 +105,13 @@ class TagListEditor(QDialog, Ui_TagListEditor):
        if not question_dialog(self, _('Are your sure?'),
            '<p>'+_('Are you certain you want to delete the following items?')+'<br>'+ct):
            return
-
+        row = self.available_tags.row(deletes[0])
        for item in deletes:
            (id,ign) = item.data(Qt.UserRole).toInt()
            self.to_delete.append(id)
            self.available_tags.takeItem(self.available_tags.row(item))

+        if row >= self.available_tags.count():
+            row = self.available_tags.count() - 1
+        if row >= 0:
+            self.available_tags.scrollToItem(self.available_tags.item(row))
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@ -123,6 +123,8 @@ class Stack(QStackedWidget): # {{{
                _('Tag Browser'), I('tags.png'),
                parent=parent, side_index=0, initial_side_size=200,
                shortcut=_('Shift+Alt+T'))
+        parent.tb_splitter.state_changed.connect(
+                        self.tb_widget.set_pane_is_visible, Qt.QueuedConnection)
        parent.tb_splitter.addWidget(self.tb_widget)
        parent.tb_splitter.addWidget(parent.cb_splitter)
        parent.tb_splitter.setCollapsible(parent.tb_splitter.other_index, False)
--- a/src/calibre/gui2/library/delegates.py
+++ b/src/calibre/gui2/library/delegates.py
@ -349,10 +349,19 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
        QStyledItemDelegate.__init__(self, parent)

    def createEditor(self, parent, option, index):
-        return EnLineEdit(parent)
+        m = index.model()
+        text = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
+        editor = CommentsDialog(parent, text)
+        editor.setWindowTitle(_("Edit template"))
+        editor.textbox.setTabChangesFocus(False)
+        editor.textbox.setTabStopWidth(20)
+        d = editor.exec_()
+        if d:
+            m.setData(index, QVariant(editor.textbox.toPlainText()), Qt.EditRole)
+        return None

    def setModelData(self, editor, model, index):
-        val = unicode(editor.text())
+        val = unicode(editor.textbox.toPlainText())
        try:
            validation_formatter.validate(val)
        except Exception, err:
@ -364,7 +373,7 @@ class CcTemplateDelegate(QStyledItemDelegate): # {{{
    def setEditorData(self, editor, index):
        m = index.model()
        val = m.custom_columns[m.column_map[index.column()]]['display']['composite_template']
-        editor.setText(val)
+        editor.textbox.setPlainText(val)


 # }}}
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@ -223,21 +223,22 @@ class BooksModel(QAbstractTableModel): # {{{
    def by_author(self):
        return self.sorted_on[0] == 'authors'

+    def books_deleted(self):
+        self.count_changed()
+        self.clear_caches()
+        self.reset()
+
    def delete_books(self, indices):
        ids = map(self.id, indices)
        for id in ids:
            self.db.delete_book(id, notify=False)
-        self.count_changed()
-        self.clear_caches()
-        self.reset()
+        self.books_deleted()
        return ids

    def delete_books_by_id(self, ids):
        for id in ids:
            self.db.delete_book(id)
-        self.count_changed()
-        self.clear_caches()
-        self.reset()
+        self.books_deleted()

    def books_added(self, num):
        if num > 0:
@ -302,6 +303,20 @@ class BooksModel(QAbstractTableModel): # {{{
        return self.rowCount(None)

    def get_book_display_info(self, idx):
+        def custom_keys_to_display():
+            ans = getattr(self, '_custom_fields_in_book_info', None)
+            if ans is None:
+                cfkeys = set(self.db.custom_field_keys())
+                yes_fields = set(tweaks['book_details_will_display'])
+                no_fields = set(tweaks['book_details_wont_display'])
+                if '*' in yes_fields:
+                    yes_fields = cfkeys
+                if '*' in no_fields:
+                    no_fields = cfkeys
+                ans = frozenset(yes_fields - no_fields)
+                setattr(self, '_custom_fields_in_book_info', ans)
+            return ans
+
        data = {}
        cdata = self.cover(idx)
        if cdata:
@ -333,7 +348,10 @@ class BooksModel(QAbstractTableModel): # {{{
                _('Book %s of %s.')%\
                    (sidx, prepare_string_for_xml(series))
        mi = self.db.get_metadata(idx)
+        cf_to_display = custom_keys_to_display()
        for key in mi.custom_field_keys():
+            if key not in cf_to_display:
+                continue
            name, val = mi.format_field(key)
            if val:
                data[name] = val
--- a/src/calibre/gui2/preferences/main.py
+++ b/src/calibre/gui2/preferences/main.py
@ -251,10 +251,28 @@ class Preferences(QMainWindow):
        self.close()
        self.run_wizard_requested.emit()

+    def set_tooltips_for_labels(self):
+
+        def process_child(child):
+            for g in child.children():
+                if isinstance(g, QLabel):
+                    buddy = g.buddy()
+                    if buddy is not None and hasattr(buddy, 'toolTip'):
+                        htext = unicode(buddy.toolTip()).strip()
+                        etext = unicode(g.toolTip()).strip()
+                        if htext and not etext:
+                            g.setToolTip(htext)
+                            g.setWhatsThis(htext)
+                else:
+                    process_child(g)
+
+        process_child(self.showing_widget)
+
    def show_plugin(self, plugin):
        self.showing_widget = plugin.create_widget(self.scroll_area)
        self.showing_widget.genesis(self.gui)
        self.showing_widget.initialize()
+        self.set_tooltips_for_labels()
        self.scroll_area.setWidget(self.showing_widget)
        self.stack.setCurrentIndex(1)
        self.showing_widget.show()
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@ -87,6 +87,13 @@ class TagsView(QTreeView): # {{{
        self.setDragDropMode(self.DropOnly)
        self.setDropIndicatorShown(True)
        self.setAutoExpandDelay(500)
+        self.pane_is_visible = False
+
+    def set_pane_is_visible(self, to_what):
+        pv = self.pane_is_visible
+        self.pane_is_visible = to_what
+        if to_what and not pv:
+            self.recount()

    def set_database(self, db, tag_match, sort_by):
        self.hidden_categories = config['tag_browser_hidden_categories']
@ -300,7 +307,7 @@ class TagsView(QTreeView): # {{{
        return self.isExpanded(idx)

    def recount(self, *args):
-        if self.disable_recounting:
+        if self.disable_recounting or not self.pane_is_visible:
            return
        self.refresh_signal_processed = True
        ci = self.currentIndex()
@ -969,6 +976,7 @@ class TagBrowserWidget(QWidget): # {{{
        self._layout.setContentsMargins(0,0,0,0)

        parent.tags_view = TagsView(parent)
+        self.tags_view = parent.tags_view
        self._layout.addWidget(parent.tags_view)

        parent.sort_by = QComboBox(parent)
@ -998,6 +1006,9 @@ class TagBrowserWidget(QWidget): # {{{
                _('Add your own categories to the Tag Browser'))
        parent.edit_categories.setStatusTip(parent.edit_categories.toolTip())

+    def set_pane_is_visible(self, to_what):
+        self.tags_view.set_pane_is_visible(to_what)
+

 # }}}

--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@ -129,6 +129,7 @@ class CoverCache(Thread): # {{{
        self.keep_running = True
        self.cache = {}
        self.lock = RLock()
+        self.allowed_ids = frozenset([])
        self.null_image = QImage()

    def stop(self):
@ -175,6 +176,11 @@ class CoverCache(Thread): # {{{
                break
            for id_ in ids:
                time.sleep(0.050) # Limit 20/second to not overwhelm the GUI
+                if not self.keep_running:
+                    return
+                with self.lock:
+                    if id_ not in self.allowed_ids:
+                        continue
                try:
                    img = self._image_for_id(id_)
                except:
@ -193,6 +199,7 @@ class CoverCache(Thread): # {{{

    def set_cache(self, ids):
        with self.lock:
+            self.allowed_ids = frozenset(ids)
            already_loaded = set([])
            for id in self.cache.keys():
                if id in ids:
@ -213,8 +220,9 @@ class CoverCache(Thread): # {{{
    def refresh(self, ids):
        with self.lock:
            for id_ in ids:
-                self.cache.pop(id_, None)
-                self.load_queue.put(id_)
+                cover = self.cache.pop(id_, None)
+                if cover is not None:
+                    self.load_queue.put(id_)
 # }}}

 ### Global utility function for get_match here and in gui2/library.py
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@ -953,23 +953,24 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
            self.notify('metadata', [id])
        return True

-    def delete_book(self, id, notify=True):
+    def delete_book(self, id, notify=True, commit=True):
        '''
        Removes book from the result cache and the underlying database.
+        If you set commit to False, you must call clean() manually afterwards
        '''
        try:
            path = os.path.join(self.library_path, self.path(id, index_is_id=True))
        except:
            path = None
-        self.data.remove(id)
        if path and os.path.exists(path):
            self.rmtree(path)
            parent = os.path.dirname(path)
            if len(os.listdir(parent)) == 0:
                self.rmtree(parent)
        self.conn.execute('DELETE FROM books WHERE id=?', (id,))
-        self.conn.commit()
-        self.clean()
+        if commit:
+            self.conn.commit()
+            self.clean()
        self.data.books_deleted([id])
        if notify:
            self.notify('delete', [id])
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@ -119,10 +119,8 @@ class SafeFormat(TemplateFormatter):
            try:
                b = self.book.get_user_metadata(key, False)
            except:
-                if DEBUG:
-                    traceback.print_exc()
+                traceback.print_exc()
                b = None
-
            if b is not None and b['datatype'] == 'composite':
                if key in self.composite_values:
                    return self.composite_values[key]
@ -135,8 +133,7 @@ class SafeFormat(TemplateFormatter):
                return val.replace('/', '_').replace('\\', '_')
            return ''
        except:
-            if DEBUG:
-                traceback.print_exc()
+            traceback.print_exc()
            return key

 def get_components(template, mi, id, timefmt='%b %Y', length=250,
@ -155,6 +152,8 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
        format_args['tags'] = mi.format_tags()
        if format_args['tags'].startswith('/'):
            format_args['tags'] = format_args['tags'][1:]
+    else:
+        format_args['tags'] = ''
    if mi.series:
        format_args['series'] = tsfmt(mi.series)
        if mi.series_index is not None:
@ -254,6 +253,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards,
        if not os.path.exists(dirpath):
            raise

+    ocover = mi.cover
    if opts.save_cover and cover and os.access(cover, os.R_OK):
        with open(base_path+'.jpg', 'wb') as f:
            with open(cover, 'rb') as s:
@ -267,6 +267,8 @@ def do_save_book_to_disk(id_, mi, cover, plugboards,
        with open(base_path+'.opf', 'wb') as f:
            f.write(opf)

+    mi.cover = ocover
+
    written = False
    for fmt in formats:
        global plugboard_save_to_disk_value, plugboard_any_format_value
--- a/src/calibre/library/server/base.py
+++ b/src/calibre/library/server/base.py
@ -43,6 +43,9 @@ class DispatchController(object): # {{{
        kwargs['action'] = 'f_%d'%len(self.funcs)
        if route != '/':
            route = self.prefix + route
+        elif self.prefix:
+            self.dispatcher.connect(name+'prefix_extra', self.prefix, self,
+                    **kwargs)
        self.dispatcher.connect(name, route, self, **kwargs)
        self.funcs.append(expose(func))

--- a/src/calibre/library/server/browse.py
+++ b/src/calibre/library/server/browse.py
@ -359,7 +359,7 @@ class BrowseServer(object):
                icon = 'blank.png'
            cats.append((meta['name'], category, icon))

-        cats = [(u'<li><a title="{2} {0}" href="/browse/category/{1}">&nbsp;</a>'
+        cats = [(u'<li><a title="{2} {0}" href="{3}/browse/category/{1}">&nbsp;</a>'
                 u'<img src="{3}{src}" alt="{0}" />'
                 u'<span class="label">{0}</span>'
                 u'</li>')
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@ -101,8 +101,8 @@ Composite columns can use any template option, including formatting.

 You cannot change the data contained in a composite column. If you edit a composite column by double-clicking on any item, you will open the template for editing, not the underlying data. Editing the template on the GUI is a quick way of testing and changing composite columns.

-Using functions in templates
-----------------------------
+Using functions in templates - single-function mode
+---------------------------------------------------

 Suppose you want to display the value of a field in upper case, when that field is normally in title case. You can do this (and many more things) using the functions available for templates. For example, to display the title in upper case, use ``{title:uppercase()}``. To display it in title case, use ``{title:titlecase()}``.

@ -137,6 +137,82 @@ Note that you can use the prefix and suffix as well. If you want the number to a
    {#myint:0>3s:ifempty(0)|[|]}
    

+Using functions in templates - program mode
+-------------------------------------------
+
+The template language program mode differs from single-function mode in that it permits you to write template expressions that refer to other metadata fields, modify values, and do arithmetic. It is a reasonably complete programming language.
+
+Beginning with an example, assume that you want your template to show the series for a book if it has one, otherwise show the value of a custom field #genre. You cannot do this in the basic language because you cannot make reference to another metadata field within a template expression. In program mode, you can. The following expression works::
+
+    {#series:'ifempty($, field('#genre'))'}
+
+The example shows several things:
+
+    * program mode is used if the expression begins with ``:'`` and ends with ``'``. Anything else is assumed to be single-function.
+    * the variable ``$`` stands for the field the expression is operating upon, ``#series`` in this case.
+    * functions must be given all their arguments. There is no default value. This is true for the standard builtin functions, and is a significant difference from single-function mode.
+    * white space is ignored and can be used anywhere within the expression.
+    * constant strings are enclosed in matching quotes, either ``'`` or ``"``.
+    
+The language is similar to ``functional`` languages in that it is built almost entirely from functions. A statement is a function. An expression is a function. Constants and identifiers can be thought of as functions returning the value indicated by the constant or stored in the identifier.
+
+The syntax of the language is shown by the following grammar::
+
+    constant   ::= " string " | ' string ' | number
+    identifier ::= sequence of letters or ``_`` characters
+    function   ::= identifier ( statement [ , statement ]* )
+    expression ::= identifier | constant | function
+    statement  ::= expression [ ; expression ]*
+    program    ::= statement
+
+An ``expression`` always has a value, either the value of the constant, the value contained in the identifier, or the value returned by a function. The value of a ``statement`` is the value of the last expression in the sequence of statements. As such, the value of the program (statement)::
+
+    1; 2; 'foobar'; 3
+    
+is 3.
+
+Another example of a complex but rather silly program might help make things clearer::
+
+    {series_index:'
+        substr(
+            strcat($, '->', 
+                cmp(divide($, 2), 1, 
+                    assign(c, 1); substr('lt123', c, 0), 
+                    'eq', 'gt')),
+            0, 6)
+       '| prefix | suffix}
+    
+This program does the following: 
+
+    * specify that the field being looked at is series_index. This sets the value of the variable ``$``.
+    * calls the ``substr`` function, which takes 3 parameters ``(str, start, end)``. It returns a string formed by extracting the start through end characters from string, zero-based (the first character is character zero). In this case the string will be computed by the ``strcat`` function, the start is 0, and the end is 6. In this case it will return the first 6 characters of the string returned by ``strcat``, which must be evaluated before substr can return.
+    * calls the ``strcat`` (string concatenation) function. Strcat accepts 1 or more arguments, and returns a string formed by concatenating all the values. In this case there are three arguments. The first parameter is the value in ``$``, which here is the value of ``series_index``. The second paremeter is the constant string ``'->'``. The third parameter is the value returned by the ``cmp`` function, which must be fully evaluated before ``strcat`` can return.
+    * The ``cmp`` function takes 5 arguments ``(x, y, lt, eq, gt)``. It compares x and y and returns the third argument ``lt`` if x < y, the fourth argument ``eq`` if x == y, and the fifth argument ``gt`` if x > y. As with all functions, all of the parameters can be statements. In this case the first parameter (the value for ``x``) is the result of dividing the series_index by 2. The second parameter ``y`` is the constant ``1``. The third parameter ``lt`` is a statement (more later). The fourth parameter ``eq`` is the constant string ``'eq'``. The fifth parameter is the constant string ``'gt'``.
+    * The third parameter (the one for ``lt``) is a statement, or a sequence of expressions. Remember that a statement (a sequence of semicolon-separated expressions) is also an expression, returning the value of the last expression in the list. In this case, the program first assigns the value ``1`` to a local variable ``c``, then returns a substring made by extracting the c'th character to the end. Since c always contains the constant ``1``, the substring will return the second through end'th characters, or ``'t123'``.
+    * Once the statement providing the value to the third parameter is executed, ``cmp`` can return a value. At that point, ``strcat` can return a value, then ``substr`` can return a value. The program then terminates.
+
+For various values of series_index, the program returns:
+
+    * series_index == undefined, result = ``prefix ->t123 suffix``
+    * series_index == 0.5, result = ``prefix 0.50-> suffix``
+    * series_index == 1, result = ``prefix 1->t12 suffix``
+    * series_index == 2, result = ``prefix 2->eq suffix``
+    * series_index == 3, result = ``prefix 3->gt suffix``
+
+All the functions listed under single-function mode can be used in program mode, noting that unlike the functions described below you must supply a first parameter providing the value the function is to act upon. 
+
+The following functions are available in addition to those described in single-function mode. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
+
+    * ``add(x, y)``	-- returns x + y. Throws an exception if either x or y are not numbers.
+    * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression
+    * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
+    * ``divide(x, y)``	-- returns x / y. Throws an exception if either x or y are not numbers.
+    * ``field(name)`` -- returns the metadata field named by ``name``.
+    * ``multiply``	-- returns x * y. Throws an exception if either x or y are not numbers.
+    * ``strcat(a, b, ...)`` -- can take any number of arguments. Returns a string formed by concatenating all the arguments.
+    * ``strcmp(x, y, lt, eq, gt)`` -- does a case-insensitive comparison x and y as strings. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``.
+    * ``substr(str, start, end)`` -- returns the ``start``'th through the ``end``'th characters of ``str``. The first character in ``str`` is the zero'th character. If end is negative, then it indicates that many characters counting from the right. If end is zero, then it indicates the last character. For example, ``substr('12345', 1, 0)`` returns ``'2345'``, and ``substr('12345', 1, -1)`` returns ``'234'``.
+    * ``subtract``	-- returns x - y. Throws an exception if either x or y are not numbers.
    
 Special notes for save/send templates
 -------------------------------------
--- a/src/calibre/translations/af.po
+++ b/src/calibre/translations/af.po
--- a/src/calibre/translations/ar.po
+++ b/src/calibre/translations/ar.po
--- a/src/calibre/translations/ast.po
+++ b/src/calibre/translations/ast.po
--- a/src/calibre/translations/bg.po
+++ b/src/calibre/translations/bg.po
--- a/src/calibre/translations/bn.po
+++ b/src/calibre/translations/bn.po
--- a/src/calibre/translations/ca.po
+++ b/src/calibre/translations/ca.po
--- a/src/calibre/translations/calibre.pot
+++ b/src/calibre/translations/calibre.pot
--- a/src/calibre/translations/cs.po
+++ b/src/calibre/translations/cs.po
--- a/src/calibre/translations/da.po
+++ b/src/calibre/translations/da.po
--- a/src/calibre/translations/de.po
+++ b/src/calibre/translations/de.po
--- a/src/calibre/translations/el.po
+++ b/src/calibre/translations/el.po
--- a/src/calibre/translations/en_AU.po
+++ b/src/calibre/translations/en_AU.po
--- a/src/calibre/translations/en_CA.po
+++ b/src/calibre/translations/en_CA.po
--- a/src/calibre/translations/en_GB.po
+++ b/src/calibre/translations/en_GB.po
--- a/src/calibre/translations/eo.po
+++ b/src/calibre/translations/eo.po
--- a/src/calibre/translations/es.po
+++ b/src/calibre/translations/es.po
--- a/src/calibre/translations/eu.po
+++ b/src/calibre/translations/eu.po
--- a/src/calibre/translations/fa.po
+++ b/src/calibre/translations/fa.po
--- a/src/calibre/translations/fi.po
+++ b/src/calibre/translations/fi.po
--- a/src/calibre/translations/fo.po
+++ b/src/calibre/translations/fo.po
--- a/src/calibre/translations/fr.po
+++ b/src/calibre/translations/fr.po
--- a/src/calibre/translations/gl.po
+++ b/src/calibre/translations/gl.po
--- a/src/calibre/translations/he.po
+++ b/src/calibre/translations/he.po
--- a/src/calibre/translations/hi.po
+++ b/src/calibre/translations/hi.po
--- a/src/calibre/translations/hr.po
+++ b/src/calibre/translations/hr.po
--- a/src/calibre/translations/hu.po
+++ b/src/calibre/translations/hu.po
--- a/src/calibre/translations/id.po
+++ b/src/calibre/translations/id.po
--- a/src/calibre/translations/it.po
+++ b/src/calibre/translations/it.po
--- a/src/calibre/translations/ja.po
+++ b/src/calibre/translations/ja.po
--- a/src/calibre/translations/ko.po
+++ b/src/calibre/translations/ko.po
--- a/src/calibre/translations/lt.po
+++ b/src/calibre/translations/lt.po
--- a/src/calibre/translations/lv.po
+++ b/src/calibre/translations/lv.po
--- a/src/calibre/translations/ml.po
+++ b/src/calibre/translations/ml.po
--- a/src/calibre/translations/mr.po
+++ b/src/calibre/translations/mr.po
--- a/src/calibre/translations/ms.po
+++ b/src/calibre/translations/ms.po
--- a/src/calibre/translations/nb.po
+++ b/src/calibre/translations/nb.po
--- a/src/calibre/translations/nds.po
+++ b/src/calibre/translations/nds.po
--- a/src/calibre/translations/nl.po
+++ b/src/calibre/translations/nl.po
--- a/src/calibre/translations/oc.po
+++ b/src/calibre/translations/oc.po
--- a/src/calibre/translations/pl.po
+++ b/src/calibre/translations/pl.po
--- a/src/calibre/translations/pt.po
+++ b/src/calibre/translations/pt.po
--- a/Show More
+++ b/Show More