Merge from trunk

.bzrignore

@@ -31,3 +31,4 @@ nbproject/
 .pydevproject
 .settings/
 *.DS_Store
+calibre_plugins/

Changelog.yaml

@@ -19,6 +19,83 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.1
+  date: 2011-05-13
+
+  new features:
+    - title: "Add Amazon DE, Beam EBooks, Beam DE, Weightless Books, Wizards Tower Books to the list of ebook stores searched by Get Books"
+
+    - title: "TXT output: All new Textile output with much greater preservation of formatting from the input document"
+
+    - title: "Migrate metadata plugin for Douban Books to the 0.8 API"
+
+    - title: "Driver for Dell Streak on windows"
+
+    - title: "Add menu items to Get Books action to search by title and author of current book"
+
+    - title: "Add title_sort as available field to CSV/XML catalogs"
+
+    - title: "Add a context menu to the manage authors dialog"
+
+    - title: "Add a button to paste isbn into the identifiers field in the edit metadata dialog automatically"
+  
+  bug fixes:
+    - title: "Amazon metadata download plugin: Fix links being stripped from comments. Also fix ratings/isbn not being parsed from kindle edition pages."
+      tickets: [782012]
+
+    - title: "Fix one source of segfaults on shutdown in the linux binary builds."
+
+    - title: "Allow the use of condensed/expanded fonts as interface fonts"
+
+    - title: "EPUB Input: Ignore missing cover file when converting, instead of erroring out."
+      tickets: [781848]
+
+    - title: "Fix custom identifier being erased by metadata download"
+      tickets: [781759]
+
+    - title: "Fix regression that broke various things when using Japanese language calibre on windows"
+      tickets: [780804]
+
+    - title: "RTF Input: Handle null color codes correctly"
+      tickets: [780728]
+
+    - title: "ODT Input: Handle inline special styles defined on <text:span> tags."
+      tickets: [780250]
+
+    - title: "Fix error when pressing next previous button with an empty search in the Plugins preferences"
+      tickets: [781135]
+
+    - title: "Ignore 'Unknown' author when downloading metadata."
+      tickets: [779348]
+
+    - title: "Fix timezone bug when setting dates in the edit metadata dialog"
+      tickets: [779497]
+
+    - title: "Fix ebook-convert not recognizing output paths starting with .."
+      tickets: [779322]
+
+  improved recipes:
+    - "Strategy+Business"
+    - Readers Digest
+    - Ming Pao
+    - Telepolis
+    - Fronda
+    - Rzeczpospolita
+
+  new recipes:
+    - title: "Various Taiwanese news sources"
+      author: Eddie Lau
+
+    - title: Replica Vedetelor, Ziua Veche 
+      author: Silviu Cotoara
+
+    - title: Welt der Physik
+      author: schuster
+
+    - title: Korea Herald
+      author: Seongkyoun Yoo
+
+
 - version: 0.8.0
   date: 2010-05-06
 

recipes/arcamax.recipe

@@ -93,7 +93,7 @@ class Arcamax(BasicNewsRecipe):
         for page in pages:
             page_soup = self.index_to_soup(url)
             if page_soup:
-                title = page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0]
+                title = self.tag_to_string(page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0])
                 page_url = url
                 # orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href']
                 prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href']
@@ -127,4 +127,3 @@ class Arcamax(BasicNewsRecipe):
                     p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
                     body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
 		'''
-

recipes/bild_de.recipe

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+    title          = u'Bild.de'
+    __author__  = 'schuster'
+    oldest_article = 1
+    max_articles_per_feed = 50
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+
+# get cover from myspace
+    cover_url = 'http://a3.l3-images.myspacecdn.com/images02/56/0232f842170b4d349779f8379c27e073/l.jpg'
+
+# set what to fetch on the site
+    remove_tags_before =  dict(name = 'h2', attrs={'id':'cover'})
+    remove_tags_after = dict(name ='div', attrs={'class':'back'})
+
+#  thanx to kiklop74 for code (see sticky thread -> Recipes - Re-usable code)
+# this one removes a lot of direct-link's
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+# remove the ad's
+    filter_regexps = [r'.\.smartadserver\.com']
+    def skip_ad_pages(self, soup):
+        return None
+
+#get the real url behind  .feedsportal.com and fetch the artikels
+    def get_article_url(self, article):
+        return article.get('id', article.get('guid', None))
+
+#list of the rss source from www.bild.de
+    feeds          = [(u'Überblick', u'http://rss.bild.de/bild.xml'),
+                          (u'News', u'http://rss.bild.de/bild-news.xml'),
+                          (u'Politik', u'http://rss.bild.de/bild-politik.xml'),
+                          (u'Unterhaltung', u'http://rss.bild.de/bild-unterhaltung.xml'),
+                          (u'Sport', u'http://rss.bild.de/bild-sport.xml'),
+                          (u'Lifestyle', u'http://rss.bild.de/bild-lifestyle.xml'),
+                          (u'Ratgeber', u'http://rss.bild.de/bild-ratgeber.xml')
+]

recipes/borse_online.recipe

@@ -0,0 +1,33 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Börse-online'
+    __author__  = 'schuster'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    cover_url = 'http://www.dpv.de/images/1995/source.gif'
+    masthead_url = 'http://www.zeitschriften-cover.de/cover/boerse-online-cover-januar-2010-x1387.jpg'
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    remove_tags_bevor = [dict(name='h3')]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelfuss'})]
+    remove_tags = [dict(attrs={'class':['moduleTopNav', 'moduleHeaderNav', 'text', 'blau', 'poll1150']}),
+                dict(id=['newsletterlayer', 'newsletterlayerClose', 'newsletterlayer_body', 'newsletterarray_error', 'newsletterlayer_emailadress', 'newsletterlayer_submit', 'kommentar']),
+                dict(name=['h2', 'Gesamtranking', 'h3',''])]
+
+    def print_version(self, url):
+        return url.replace('.html#nv=rss', '.html?mode=print')
+
+
+
+    feeds          = [(u'Börsennachrichten', u'http://www.boerse-online.de/rss/')]
+

recipes/capital_de.recipe

@@ -0,0 +1,61 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1305470859(BasicNewsRecipe):
+    title                 = u'Capital.de'
+    language              = 'de'
+    __author__            = 'schuster'
+    oldest_article        =7
+    max_articles_per_feed = 35
+    no_stylesheets        = True
+    remove_javascript     = True
+    use_embedded_content  = False
+    masthead_url = 'http://www.wirtschaftsmedien-shop.de/media/stores/wirtschaftsmedien/capital/teaser_large_abo.jpg'
+    cover_url = 'http://d1kb9jvg6ylufe.cloudfront.net/WebsiteCMS/de/unternehmen/linktipps/mainColumn/08/image/DE_Capital_bis20mm_SW.jpg'
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    def print_version(self, url):
+        return url.replace ('nv=rss#utm_source=rss2&utm_medium=rss_feed&utm_campaign=/', 'mode=print')
+    remove_tags_bevor = [dict(name='td', attrs={'class':'textcell'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelsplit'})]
+
+    feeds = [   (u'Wirtschaftsmagazin', u'http://www.capital.de/rss/'),
+	(u'Unternehmen', u'http://www.capital.de/rss/unternehmen'),
+	(u'Finanz & Geldanlage', u'http://www.capital.de/rss/finanzen/geldanlage')]
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('div',attrs={'class':'artikelsplit'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'printable'})
+           for it in texttag.findAll(style=True):
+               del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('div', attrs={'class':'artikelsplit'}):
+            item.extract()
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'class':'artikelsplit'})
+        if pager:
+           pager.extract()
+        return self.adeify_images(soup)
+
+
+
+    remove_tags = [dict(attrs={'class':['navSeitenAlle', 'kommentieren', 'teaserheader', 'teasercontent', 'info', 'zwischenhead', 'artikelsplit']}),
+                dict(id=['topNav', 'mainNav', 'subNav', 'socialmedia', 'footerRahmen', 'gatrixx_marktinformationen', 'pager', 'weitere']),
+                dict(span=['ratingtext', 'Gesamtranking', 'h3','']),
+	dict(rel=['canonical'])]
+

recipes/china_times.recipe

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+# dug from http://www.mobileread.com/forums/showthread.php?p=1012294
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277443634(BasicNewsRecipe):
+    title = u'中時電子報'
+    oldest_article = 1
+    max_articles_per_feed = 100
+
+    feeds = [(u'焦點', u'http://rss.chinatimes.com/rss/focus-u.rss'),
+             (u'政治', u'http://rss.chinatimes.com/rss/Politic-u.rss'),
+             (u'社會', u'http://rss.chinatimes.com/rss/social-u.rss'),
+             (u'國際', u'http://rss.chinatimes.com/rss/international-u.rss'),
+             (u'兩岸', u'http://rss.chinatimes.com/rss/mainland-u.rss'),
+             (u'地方', u'http://rss.chinatimes.com/rss/local-u.rss'),
+             (u'言論', u'http://rss.chinatimes.com/rss/comment-u.rss'),
+             (u'科技', u'http://rss.chinatimes.com/rss/technology-u.rss'),
+             (u'運動', u'http://rss.chinatimes.com/rss/sport-u.rss'),
+             (u'藝文', u'http://rss.chinatimes.com/rss/philology-u.rss'),
+             #(u'旺報', u'http://rss.chinatimes.com/rss/want-u.rss'),
+             #(u'財經', u'http://rss.chinatimes.com/rss/finance-u.rss'),  # broken links
+             #(u'股市', u'http://rss.chinatimes.com/rss/stock-u.rss')  # broken links
+             ]
+
+    __author__ = 'einstuerzende, updated by Eddie Lau'
+    __version__ = '1.0'
+    language = 'zh'
+    publisher = 'China Times Group'
+    description = 'China Times (Taiwan)'
+    category = 'News, Chinese, Taiwan'
+    remove_javascript = True
+    use_embedded_content = False
+    no_stylesheets = True
+    encoding = 'big5'
+    conversion_options = {'linearize_tables':True}
+    masthead_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif'
+    cover_url = 'http://www.fcuaa.org/gif/chinatimeslogo.gif'
+    keep_only_tags = [dict(name='div', attrs={'class':['articlebox','articlebox clearfix']})]
+    remove_tags = [dict(name='div', attrs={'class':['focus-news']})]
+

recipes/cosmopolitan_de.recipe

@@ -0,0 +1,34 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305567197(BasicNewsRecipe):
+    title          = u'Cosmopolitan.de'
+    __author__  = 'schuster'
+    oldest_article = 7
+    language = 'de'
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_javascript      = True
+    cover_url = 'http://www.cosmopolitan.com/cm/shared/site_images/print_this/cosmopolitan_logo.gif'
+    remove_tags_before =  dict(name = 'h1', attrs={'class':'artikel'})
+    remove_tags_after = dict(name ='div', attrs={'class':'morePages'})
+    extra_css = '''
+                    h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
+                    h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
+                '''
+    remove_tags = [    dict(id='strong'),
+           dict(title='strong'),
+                                           dict(name='span'),
+           dict(name='li', attrs={'class':'large'}),
+           dict(name='ul', attrs={'class':'articleImagesPortrait clearfix'}),
+           dict(name='p', attrs={'class':'external'}),
+           dict(name='a', attrs={'target':'_blank'}),]
+    feeds          = [  (u'Komplett', u'http://www.cosmopolitan.de/rss/allgemein.xml'),
+        (u'Mode', u'http://www.cosmopolitan.de/rss/mode.xml'),
+        (u'Beauty', u'http://www.cosmopolitan.de/rss/beauty.xml'),
+        (u'Liebe&Sex', u'http://www.cosmopolitan.de/rss/liebe.xml'),
+        (u'Psychologie', u'http://www.cosmopolitan.de/rss/psychologie.xml'),
+        (u'Job&Karriere', u'http://www.cosmopolitan.de/rss/job.xml'),
+        (u'Lifestyle', u'http://www.cosmopolitan.de/rss/lifestyle.xml'),
+        (u'Shopping', u'http://www.cosmopolitan.de/rss/shopping.xml'),
+        (u'Bildergalerien', u'http://www.cosmopolitan.de/rss/bildgalerien.xml')]

recipes/dilemaveche.recipe

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+dilemaveche.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DilemaVeche(BasicNewsRecipe):
+    title                 = u'Dilema Veche'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Sunt vechi, domnule!'
+    publisher             = u'Dilema Veche'
+    oldest_article        = 50
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare'
+    encoding              = 'utf-8'
+    cover_url             = 'http://www.dilemaveche.ro/sites/all/themes/dilema/theme/dilema_two/layouter/dilema_two_homepage/logo.png'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                          dict(name='h1', attrs={'class':'art_title'})
+                        , dict(name='h1', attrs={'class':'art_title online'})
+                        , dict(name='div', attrs={'class':'item'})
+                        , dict(name='div', attrs={'class':'art_content'})
+                     ]
+
+    remove_tags = [
+                         dict(name='div', attrs={'class':['article_details']})
+                        , dict(name='div', attrs={'class':['controale']})
+                        , dict(name='div', attrs={'class':['art_related_left']})
+                  ]
+
+    remove_tags_after = [
+                            dict(name='div', attrs={'class':['article_details']})
+                        ]
+
+    feeds          = [
+                        (u'Feeds', u'http://www.dilemaveche.ro/rss.xml')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/divahair.recipe

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+divahair.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DivaHair(BasicNewsRecipe):
+    title                 = u'Diva Hair'
+    language              = 'ro'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Coafuri, frizuri, tunsori ..'
+    publisher             = u'Diva Hair'
+    category              = u'Ziare,Stiri,Coafuri,Femei'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    remove_javascript     = True
+    cover_url             = 'http://www.divahair.ro/imgs/logo.jpg'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                         dict(name='td', attrs={'class':'spatiuart'})
+                        , dict(name='div', attrs={'class':'spatiuart'})
+                     ]
+
+
+    remove_tags = [
+                     dict(name='div', attrs={'class':'categorie'})
+                     , dict(name='div', attrs={'class':'gri gri2 detaliiart'})
+                     , dict(name='div', attrs={'class':'articol_box_bottom'})
+                  ]
+
+    remove_tags_after = [
+                             dict(name='div', attrs={'class':'articol_box_bottom'})
+                        ]
+
+    feeds = [ (u'\u0218tiri', u'http://www.divahair.ro/feed') ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/dn_se.recipe

@@ -37,7 +37,7 @@ class DN_se(BasicNewsRecipe):
               ,(u'Kultur' , u'http://www.dn.se/kultur-rss'       )
             ]
 
-    keep_only_tags = [dict(name='div', attrs={'id':'article'})]
+    keep_only_tags = [dict(name='div', attrs={'id':'article-content'})]
     remove_tags_before = dict(name='h1')
     remove_tags_after = dict(name='div',attrs={'id':'byline'})
     remove_tags = [

recipes/dvhn.recipe

@@ -1,19 +1,21 @@
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1302341394(BasicNewsRecipe):
     title          = u'DvhN'
-    oldest_article = 1
+    __author__ = 'Reijndert'
+    oldest_article = 7
     max_articles_per_feed = 200
 
-    __author__ = 'Reijndert'
     no_stylesheets = True
-    cover_url = 'http://www.dvhn.nl/template/Dagblad_v2.0/gfx/logo_DvhN.gif'
+    cover_url = 'http://members.home.nl/apm.de.haas/calibre/DvhN.jpg'
     language = 'nl'
     country = 'NL'
     version = 1
     publisher = u'Dagblad van het Noorden'
     category = u'Nieuws'
     description = u'Nieuws uit Noord Nederland'
+    timefmt = ' %Y-%m-%d (%a)'
 
 
     keep_only_tags = [dict(name='div', attrs={'id':'fullPicture'})
@@ -21,11 +23,26 @@ class AdvancedUserRecipe1302341394(BasicNewsRecipe):
                   ]
 
     remove_tags = [
-                    dict(name=['object','link','iframe','base'])
-                    ,dict(name='span',attrs={'class':'copyright'})
+                    dict(name='span',attrs={'class':'location'})
                 ]
 
-    feeds          = [(u'Drenthe', u'http://www.dvhn.nl/nieuws/drenthe/index.jsp?service=rss'), (u'Groningen', u'http://www.dvhn.nl/nieuws/groningen/index.jsp?service=rss'), (u'Nederland', u'http://www.dvhn.nl/nieuws/nederland/index.jsp?service=rss'), (u'Wereld', u'http://www.dvhn.nl/nieuws/wereld/index.jsp?service=rss'), (u'Economie', u'http://www.dvhn.nl/nieuws/economie/index.jsp?service=rss'), (u'Sport', u'http://www.dvhn.nl/nieuws/sport/index.jsp?service=rss'), (u'Cultuur', u'http://www.dvhn.nl/nieuws/kunst/index.jsp?service=rss'), (u'24 Uur', u'http://www.dvhn.nl/nieuws/24uurdvhn/index.jsp?service=rss&selectiontype=last24hours')]
+    preprocess_regexps = [
+        (re.compile(r'<a.*?>'), lambda h1: '')
+       ,(re.compile(r'</a>'), lambda h2: '')
+       ,(re.compile(r'Word vriend van Dagblad van het Noorden op Facebook'), lambda h3: '')
+       ,(re.compile(r'Volg Dagblad van het Noorden op Twitter'), lambda h3: '')
+     ]
+
+
+    feeds          = [(u'Drenthe', u'http://www.dvhn.nl/nieuws/drenthe/index.jsp?service=rss')
+	, (u'Groningen', u'http://www.dvhn.nl/nieuws/groningen/index.jsp?service=rss')
+	, (u'Nederland', u'http://www.dvhn.nl/nieuws/nederland/index.jsp?service=rss')
+	, (u'Wereld', u'http://www.dvhn.nl/nieuws/wereld/index.jsp?service=rss')
+	, (u'Economie', u'http://www.dvhn.nl/nieuws/economie/index.jsp?service=rss')
+	, (u'Sport', u'http://www.dvhn.nl/nieuws/sport/index.jsp?service=rss')
+	, (u'Cultuur', u'http://www.dvhn.nl/nieuws/kunst/index.jsp?service=rss')
+	, (u'24 Uur', u'http://www.dvhn.nl/nieuws/24uurdvhn/index.jsp?service=rss&selectiontype=last24hours')
+     ]
 
     extra_css = '''
                 body {font-family: verdana, arial, helvetica, geneva, sans-serif;}

recipes/economist.recipe

@@ -20,7 +20,7 @@ class Economist(BasicNewsRecipe):
     INDEX = 'http://www.economist.com/printedition'
     description = ('Global news and current affairs from a European'
             ' perspective. Best downloaded on Friday mornings (GMT)')
-
+    extra_css      = '.headline {font-size: x-large;} \n h2 { font-size: small;  } \n h1 { font-size: medium;  }'
     oldest_article = 7.0
     cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg'
     remove_tags = [

recipes/express_de.recipe

@@ -0,0 +1,74 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Express.de'
+    __author__  = 'schuster'
+    oldest_article = 2
+    max_articles_per_feed = 50
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    extra_css = '''
+                    h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
+                    h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
+
+                '''
+    remove_javascript      = True
+    remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'MoreNews'})]
+
+    remove_tags = [dict(id='kalaydo'),
+           dict(id='Header'),
+           dict(id='Searchline'),
+           dict(id='MainNav'),
+           dict(id='Logo'),
+           dict(id='MainLinkSpacer'),
+           dict(id='MainLinks'),
+           dict(title='Diese Seite Bookmarken'),
+
+                                           dict(name='span'),
+           dict(name='div', attrs={'class':'spacer_leftneu'}),
+           dict(name='div', attrs={'class':'button kalaydologo'}),
+           dict(name='div', attrs={'class':'button stellenneu'}),
+           dict(name='div', attrs={'class':'button autoneu'}),
+           dict(name='div', attrs={'class':'button immobilienneu'}),
+           dict(name='div', attrs={'class':'button kleinanzeigen'}),
+           dict(name='div', attrs={'class':'button tiereneu'}),
+           dict(name='div', attrs={'class':'button ferienwohnungen'}),
+           dict(name='div', attrs={'class':'button inserierenneu'}),
+           dict(name='div', attrs={'class':'spacer_rightneu'}),
+           dict(name='div', attrs={'class':'spacer_rightcorner'}),
+           dict(name='div', attrs={'class':'HeaderMetaNav'}),
+           dict(name='div', attrs={'class':'HeaderSearchOption'}),
+           dict(name='div', attrs={'class':'HeaderSearch'}),
+           dict(name='div', attrs={'class':'sbutton'}),
+           dict(name='div', attrs={'class':'active'}),
+
+]
+
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds          = [(u'Top-Themen', u'http://www.express.de/home/-/2126/2126/-/view/asFeed/-/index.xml'),
+                          (u'Regional - Köln', u'http://www.express.de/regional/koeln/-/2856/2856/-/view/asFeed/-/index.xml'),
+                          (u'Regional - Bonn', u'http://www.express.de/regional/bonn/-/2860/2860/-/view/asFeed/-/index.xml'),
+                          (u'Regional - Düsseldorf', u'http://www.express.de/regional/duesseldorf/-/2858/2858/-/view/asFeed/-/index.xml'),
+                          (u'Regional - Region', u'http://www.express.de/regional/-/2178/2178/-/view/asFeed/-/index.xml'),
+                          (u'Sport-News', u'http://www.express.de/sport/-/2176/2176/-/view/asFeed/-/index.xml'),
+                          (u'Fussball-News', u'http://www.express.de/sport/fussball/-/3186/3186/-/view/asFeed/-/index.xml'),
+                          (u'1.FC Köln News', u'http://www.express.de/sport/fussball/fc-koeln/-/3192/3192/-/view/asFeed/-/index.xml'),
+                          (u'Alemannia Aachen News', u'http://www.express.de/sport/fussball/alemannia/-/3290/3290/-/view/asFeed/-/index.xml'),
+                          (u'Borussia M~Gladbach', u'http://www.express.de/sport/fussball/gladbach/-/3286/3286/-/view/asFeed/-/index.xml'),
+                          (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'),
+                          (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'),
+                          (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'),
+
+
+
+]

recipes/faznet.recipe

@@ -1,51 +1,38 @@
-__license__   = 'GPL v3'
-__copyright__ = '2008-2009, Kovid Goyal <kovid at kovidgoyal.net>, Darko Miletic <darko at gmail.com>'
-'''
-Profile to download FAZ.net
-'''
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
-from calibre.web.feeds.news import BasicNewsRecipe
- 
-class FazNet(BasicNewsRecipe): 
-    title                 = 'FAZ NET'
-    __author__            = 'Kovid Goyal, Darko Miletic'
+    title          = u'Faz.net'
+    __author__  = 'schuster'
+    remove_tags = [dict(attrs={'class':['right', 'ArrowLinkRight', 'ModulVerlagsInfo', 'left', 'Head']}),
+                dict(id=['BreadCrumbs', 'tstag', 'FazFooterPrint']),
+                dict(name=['script', 'noscript', 'style'])]
+    oldest_article = 2
     description           = 'Frankfurter Allgemeine Zeitung'
-    publisher             = 'FAZ Electronic Media GmbH'
-    category              = 'news, politics, Germany'
-    use_embedded_content  = False
-    language = 'de'
- 
-    max_articles_per_feed = 30 
-    no_stylesheets        = True
-    encoding              = 'utf-8'
-    remove_javascript     = True
-
-    html2lrf_options = [
-                          '--comment', description
-                        , '--category', category
-                        , '--publisher', publisher
-                        ]
-    
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
-    
-    keep_only_tags = [dict(name='div', attrs={'class':'Article'})]
-
-    remove_tags = [
-                     dict(name=['object','link','embed','base'])
-                    ,dict(name='div', attrs={'class':['LinkBoxModulSmall','ModulVerlagsInfo']})
-                  ]
-    
-
-    feeds = [ ('FAZ.NET', 'http://www.faz.net/s/Rub/Tpl~Epartner~SRss_.xml') ] 
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    cover_url = 'http://www.faz.net/f30/Images/Logos/logo.gif'
 
     def print_version(self, url):
-        article, sep, rest = url.partition('?')    
-        return article.replace('.html', '~Afor~Eprint.html') 
+        return url.replace('.html', '~Afor~Eprint.html')
+
+
+
+    feeds          = [(u'Politik', u'http://www.faz.net/s/RubA24ECD630CAE40E483841DB7D16F4211/Tpl~Epartner~SRss_.xml'),
+          (u'Wirtschaft', u'http://www.faz.net/s/RubC9401175958F4DE28E143E68888825F6/Tpl~Epartner~SRss_.xml'),
+          (u'Feuilleton', u'http://www.faz.net/s/RubCC21B04EE95145B3AC877C874FB1B611/Tpl~Epartner~SRss_.xml'),
+          (u'Sport', u'http://www.faz.net/s/Rub9F27A221597D4C39A82856B0FE79F051/Tpl~Epartner~SRss_.xml'),
+          (u'Gesellschaft', u'http://www.faz.net/s/Rub02DBAA63F9EB43CEB421272A670A685C/Tpl~Epartner~SRss_.xml'),
+          (u'Finanzen', u'http://www.faz.net/s/Rub4B891837ECD14082816D9E088A2D7CB4/Tpl~Epartner~SRss_.xml'),
+          (u'Wissen', u'http://www.faz.net/s/Rub7F4BEE0E0C39429A8565089709B70C44/Tpl~Epartner~SRss_.xml'),
+          (u'Reise', u'http://www.faz.net/s/RubE2FB5CA667054BDEA70FB3BC45F8D91C/Tpl~Epartner~SRss_.xml'),
+          (u'Technik & Motor', u'http://www.faz.net/s/Rub01E4D53776494844A85FDF23F5707AD8/Tpl~Epartner~SRss_.xml'),
+          (u'Beruf & Chance', u'http://www.faz.net/s/RubB1E10A8367E8446897468EDAA6EA0504/Tpl~Epartner~SRss_.xml'),
+          (u'Kunstmarkt', u'http://www.faz.net/s/RubBC09F7BF72A2405A96718ECBFB68FBFE/Tpl~Epartner~SRss_.xml'),
+          (u'Immobilien ', u'http://www.faz.net/s/RubFED172A9E10F46B3A5F01B02098C0C8D/Tpl~Epartner~SRss_.xml'),
+          (u'Rhein-Main Zeitung', u'http://www.faz.net/s/RubABE881A6669742C2A5EBCB5D50D7EBEE/Tpl~Epartner~SRss_.xml'),
+          (u'Atomdebatte ', u'http://www.faz.net/s/Rub469C43057F8C437CACC2DE9ED41B7950/Tpl~Epartner~SRss_.xml')
+    ]
 
-    def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
-        soup.head.insert(0,mtag)        
-        del soup.body['onload']
-        for item in soup.findAll(style=True):
-            del item['style']
-        return soup

recipes/financialsense.recipe

@@ -0,0 +1,64 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.financialsense.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class FinancialSense(BasicNewsRecipe):
+    title                 = 'Financial Sense'
+    __author__            = 'Darko Miletic'
+    description           = 'Uncommon News & Views for the Wise Investor'
+    publisher             = 'Financial Sense'
+    category              = 'news, finances, politics, USA'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://www.financialsense.com/sites/default/files/logo.jpg'
+    extra_css             = """
+                               body{font-family: Arial,"Helvetica Neue",Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               h2{color: gray}
+                               .name{margin-right: 5em}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags      =[dict(name=['meta','link','base','object','embed','iframe'])]
+    remove_tags_after=dict(attrs={'class':'vcard'})
+    keep_only_tags   =[dict(attrs={'class':['title','post-meta','content','item-title','vcard']})]
+    remove_attributes=['lang','type']
+
+
+    feeds = [(u'Articles', u'http://feeds.feedburner.com/fso')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/glamour.recipe

@@ -0,0 +1,38 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305547242(BasicNewsRecipe):
+    title          = u'Glamour (US)'
+    oldest_article = 21
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language = 'en'
+    remove_javascript      = True
+    __author__  = 'Anonymous'
+    remove_tags = [dict(name='div', attrs={'class':'articles_footer', 'class':'printoptions'})]
+
+    def print_version(self, url):
+        return url + '?printable=true'
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds  = [	(u'All Fashion', u'http://feeds.glamour.com/glamour/all_fashion'),
+                    (u'All Beauty', u'http://feeds.glamour.com/glamour/all_beauty'),
+                    (u'All Sex, Love & Life', u'http://feeds.glamour.com/glamour/sex_love_life'),
+                    (u'All Health & Fitness', u'http://feeds.glamour.com/glamour/health_fitness'),
+                    (u'Shopping', u'http://feeds.glamour.com/glamour/shopping'),
+                    (u'Slaves to Fashion blog', u'http://feeds.glamour.com/glamour/slavestofashion'),
+                    (u'The Girls in the Beauty Department', u'http://feeds.glamour.com/glamour/thegirlsinthebeautydepartment'),
+                    (u'Smitten blog', u'http://feeds.glamour.com/glamour/smitten'),
+                    (u'Save the Date', u'http://feeds.feedburner.com/glamour/save-the-date'),
+                    (u'Single-ish blog', u'http://feeds.glamour.com/glamour/glamoursingle-ish'),
+                    (u'Save the Date', u'http://feeds.feedburner.com/glamour/save-the-date'),
+                    (u'Vitamin G blog', u'http://feeds.glamour.com/glamour/vitamin-g'),
+                    (u'Margarita Shapes Up blog', u'http://feeds.glamour.com/glamour/margaritashapesup'),
+                    (u'Little Miss Fortune blog', u'http://feeds.glamour.com/glamour/little-miss-fortune'),
+ ]

recipes/go_comics.recipe

@@ -6,13 +6,13 @@ __copyright__ = 'Copyright 2010 Starson17'
 www.gocomics.com
 '''
 from calibre.web.feeds.news import BasicNewsRecipe
-import mechanize
+import mechanize, re
 
 class GoComics(BasicNewsRecipe):
     title               = 'GoComics'
     __author__          = 'Starson17'
-    __version__         = '1.03'
-    __date__            = '09 October 2010'
+    __version__         = '1.05'
+    __date__            = '19 may 2011'
     description         = u'200+ Comics - Customize for more days/comics: Defaults to 7 days, 25 comics - 20 general, 5 editorial.'
     category            = 'news, comics'
     language            = 'en'
@@ -20,6 +20,7 @@ class GoComics(BasicNewsRecipe):
     no_stylesheets      = True
     remove_javascript   = True
     cover_url           = 'http://paulbuckley14059.files.wordpress.com/2008/06/calvin-and-hobbes.jpg'
+    remove_attributes = ['style']
 
     ####### USER PREFERENCES - COMICS, IMAGE SIZE AND NUMBER OF COMICS TO RETRIEVE ########
     # num_comics_to_get - I've tried up to 99 on Calvin&Hobbes
@@ -40,6 +41,8 @@ class GoComics(BasicNewsRecipe):
 
     remove_tags = [dict(name='a', attrs={'class':['beginning','prev','cal','next','newest']}),
                    dict(name='div', attrs={'class':['tag-wrapper']}),
+                   dict(name='a', attrs={'href':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
+                   dict(name='img', attrs={'src':re.compile(r'.*mutable_[0-9]+', re.IGNORECASE)}),
                    dict(name='ul', attrs={'class':['share-nav','feature-nav']}),
                    ]
 

recipes/golem_de.recipe

@@ -1,83 +1,70 @@
-#!/usr/bin/env  python
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
 
-from calibre.web.feeds.news import BasicNewsRecipe
-class golem_ger(BasicNewsRecipe):
     title          = u'Golem.de'
-    language = 'de'
-    __author__ = 'Kovid Goyal'
+    __author__  = 'schuster'
+
     oldest_article = 7
-    max_articles_per_feed = 100
-    language              = 'de'
-    lang                  = 'de-DE'
-    no_stylesheets        = True
-    encoding              = 'iso-8859-1'
-    recursions = 1
-    match_regexps = [r'http://www.golem.de/.*.html']
-
-    keep_only_tags     = [
-                               dict(name='h1', attrs={'class':'artikelhead'}),
-                               dict(name='p', attrs={'class':'teaser'}),
-                               dict(name='div', attrs={'class':'artikeltext'}),
-                               dict(name='h2', attrs={'id':'artikelhead'}),
-                            ]
-
-
-
-    remove_tags = [
-                    dict(name='div', attrs={'id':['similarContent','topContentWrapper','storycarousel','aboveFootPromo','comments','toolbar','breadcrumbs','commentlink','sidebar','rightColumn']}),
-                    dict(name='div', attrs={'class':['gg_embeddedSubText','gg_embeddedIndex gg_solid','gg_toOldGallery','golemGallery']}),
-                    dict(name='img', attrs={'class':['gg_embedded','gg_embeddedIconRight gg_embeddedIconFS gg_cursorpointer']}),
-                    dict(name='td', attrs={'class':['xsmall']}),
-                    ]
-
-
-    # remove_tags_after  = [
-      #                      dict(name='div', attrs={'id':['contentad2']})
-       #                 ]
-
-
-    feeds          = [
-                      (u'Golem.de', u'http://rss.golem.de/rss.php?feed=ATOM1.0'),
-                      (u'Audio/Video', u'http://rss.golem.de/rss.php?tp=av&feed=RSS2.0'),
-                      (u'Foto', u'http://rss.golem.de/rss.php?tp=foto&feed=RSS2.0'),
-                      (u'Games', u'http://rss.golem.de/rss.php?tp=games&feed=RSS2.0'),
-                      (u'Internet', u'http://rss.golem.de/rss.php?tp=inet&feed=RSS1.0'),
-                      (u'Mobil', u'http://rss.golem.de/rss.php?tp=mc&feed=ATOM1.0'),
-                      (u'Internet', u'http://rss.golem.de/rss.php?tp=inet&feed=RSS1.0'),
-                      (u'Politik/Recht', u'http://rss.golem.de/rss.php?tp=pol&feed=ATOM1.0'),
-                      (u'Desktop-Applikationen', u'http://rss.golem.de/rss.php?tp=apps&feed=RSS2.0'),
-                      (u'Software-Entwicklung', u'http://rss.golem.de/rss.php?tp=dev&feed=RSS2.0'),
-                      (u'Wirtschaft', u'http://rss.golem.de/rss.php?tp=wirtschaft&feed=RSS2.0'),
-                      (u'Hardware', u'http://rss.golem.de/rss.php?r=hw&feed=RSS2.0'),
-                      (u'Software', u'http://rss.golem.de/rss.php?r=sw&feed=RSS2.0'),
-                      (u'Networld', u'http://rss.golem.de/rss.php?r=nw&feed=RSS2.0'),
-                      (u'Entertainment', u'http://rss.golem.de/rss.php?r=et&feed=RSS2.0'),
-                      (u'TK', u'http://rss.golem.de/rss.php?r=tk&feed=RSS2.0'),
-                      (u'E-Commerce', u'http://rss.golem.de/rss.php?r=ec&feed=RSS2.0'),
-                      (u'Unternehmen/Maerkte', u'http://rss.golem.de/rss.php?r=wi&feed=RSS2.0')
-                      ]
-
-
-
-
-    feeds          = [
-                      (u'Golem.de', u'http://rss.golem.de/rss.php?feed=ATOM1.0'),
-                      (u'Mobil', u'http://rss.golem.de/rss.php?tp=mc&feed=feed=RSS2.0'),
-                      (u'OSS', u'http://rss.golem.de/rss.php?tp=oss&feed=RSS2.0'),
-                      (u'Politik/Recht', u'http://rss.golem.de/rss.php?tp=pol&feed=RSS2.0'),
-                      (u'Desktop-Applikationen', u'http://rss.golem.de/rss.php?tp=apps&feed=RSS2.0'),
-                      (u'Software-Entwicklung', u'http://rss.golem.de/rss.php?tp=dev&feed=RSS2.0'),
-                      ]
-
-
+    max_articles_per_feed = 10
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    cover_url = 'http://www.e-energy.de/images/logo_golem.jpg'
+    masthead_url = 'http://www.golem.de/staticrl/images/logo.png'
     extra_css = '''
-                h1 {color:#0066CC;font-family:Arial,Helvetica,sans-serif; font-size:30px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:20px;margin-bottom:2 em;}
-                h2 {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:22px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:16px; }
-                h3 {color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:x-small; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal; line-height:5px;}
-                h4 {color:#333333; font-family:Arial,Helvetica,sans-serif;font-size:13px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:13px; }
-                h5 {color:#333333; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; line-height:11px; text-transform:uppercase;}
-                .teaser {font-style:italic;font-size:12pt;margin-bottom:15pt;}
-                .xsmall{font-style:italic;font-size:x-small;}
-                .td{font-style:italic;font-size:x-small;}
-                img {align:left;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;}
+                    h1{ font-family:Arial,Helvetica,sans-serif;  font-size:x-large; font-weight:bold;}
+
                 '''
+    remove_javascript      = True
+    remove_tags_befor = [dict(name='header', attrs={'class':'cluster-header'})]
+    remove_tags_after = [dict(name='p', attrs={'class':'meta'})]
+    remove_tags = [dict(rel='nofollow'),
+           dict(name='header', attrs={'id':'header'}),
+           dict(name='div', attrs={'class':'dh1'}),
+           dict(name='label', attrs={'class':'implied'}),
+           dict(name='section', attrs={'id':'comments'}),
+           dict(name='li', attrs={'class':'gg_prebackcounterItem'}),
+           dict(name='li', attrs={'class':'gg_prebackcounterItem gg_embeddedIndexCounter'}),
+           dict(name='img', attrs={'class':'gg_embeddedIconRight gg_embeddedIconFS gg_cursorpointer'}),
+           dict(name='div', attrs={'target':'_blank'})
+]
+
+    def get_browser(self, *args, **kwargs):
+       from calibre import browser
+       kwargs['user_agent'] = 'mozilla'
+       return browser(*args, **kwargs)
+
+    def get_article_url(self, article):
+        return article.get('id', article.get('guid', None))
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds          = [(u'Audio/Video', u'http://rss.golem.de/rss.php?tp=av&feed=RSS2.0'),
+                          (u'Foto', u'http://rss.golem.de/rss.php?tp=foto&feed=RSS2.0'),
+                          (u'Games', u'http://rss.golem.de/rss.php?tp=games&feed=RSS2.0'),
+                          (u'Handy', u'http://rss.golem.de/rss.php?tp=handy&feed=RSS2.0'),
+                          (u'Internet', u'http://rss.golem.de/rss.php?tp=inet&feed=RSS2.0'),
+                          (u'Mobile', u'http://rss.golem.de/rss.php?tp=mc&feed=RSS2.0'),
+                          (u'OSS', u'http://rss.golem.de/rss.php?tp=oss&feed=RSS2.0'),
+                          (u'Politik/Recht', u'http://rss.golem.de/rss.php?tp=pol&feed=RSS2.0'),
+                          (u'Security', u'http://rss.golem.de/rss.php?tp=sec&feed=RSS2.0'),
+                          (u'Desktop-Applikationen', u'http://rss.golem.de/rss.php?tp=apps&feed=RSS2.0'),
+                          (u'Software-Entwicklung', u'http://rss.golem.de/rss.php?tp=dev&feed=RSS2.0'),
+                          (u'Wirtschaft', u'http://rss.golem.de/rss.php?tp=wirtschaft&feed=RSS2.0'),
+                          (u'Hardware', u'http://rss.golem.de/rss.php?r=hw&feed=RSS2.0'),
+                          (u'Software', u'http://rss.golem.de/rss.php?r=sw&feed=RSS2.0'),
+                          (u'Networld', u'http://rss.golem.de/rss.php?r=nw&feed=RSS2.0'),
+                          (u'Entertainment', u'http://rss.golem.de/rss.php?r=et&feed=RSS2.0'),
+                          (u'TK', u'http://rss.golem.de/rss.php?r=tk&feed=RSS2.0'),
+                          (u'Wirtschaft', u'http://rss.golem.de/rss.php?r=wi&feed=RSS2.0'),
+                          (u'E-Commerce', u'http://rss.golem.de/rss.php?r=ec&feed=RSS2.0')
+
+]
+

recipes/good_house_keeping.recipe

@@ -0,0 +1,31 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305547242(BasicNewsRecipe):
+    title          = u'Good House Keeping'
+    language = 'en'
+    __author__ = 'Anonymous'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_javascript      = True
+
+    def print_version(self,url):
+          segments = url.split('/')
+          printURL = '/'.join(segments[0:3]) + '/print-this/' + '/'.join(segments[4:])
+          return printURL
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds  = [	(u'Recipes & Entertaining', u'http://www.goodhousekeeping.com/food/food-rss/?src=rss'),
+	(u'Home & House', u'http://www.goodhousekeeping.com/home/home-rss/?src=rss'),
+	(u'Diet & Health', u'http://www.goodhousekeeping.com/health/health-rss/?src=rss'),
+	(u'Beauty & Style', u'http://www.goodhousekeeping.com/beauty/beauty-rss/?src=rss'),
+	(u'Family & Pets', u'http://www.goodhousekeeping.com/family/family-rss/?src=rss'),
+	(u'Saving Money', u'http://www.goodhousekeeping.com/money/money-rss/?src=rss'),
+	]

recipes/good_to_know.recipe

@@ -0,0 +1,32 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305547242(BasicNewsRecipe):
+    title          = u'Good to Know (uk)'
+    oldest_article = 14
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    remove_javascript      = True
+    __author__  = 'Anonymous'
+    language = 'en_GB'
+    remove_tags = [dict(name='div', attrs={'class':'articles_footer', 'class':'printoptions'})]
+
+    def print_version(self, url):
+        return url + '/print/1'
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    feeds  = [	(u'Family Conception Advice', u'http://www.goodtoknow.co.uk/feeds/family.rss'),
+	(u'Family Health Advice', u'http://www.goodtoknow.co.uk/feeds/health.rss'),
+	(u'Diet Advice', u'http://www.goodtoknow.co.uk/feeds/diet.rss'),
+                    (u'Food Advice', u'http://www.goodtoknow.co.uk/feeds/food.rss'),
+                    (u'Sex Advice', u'http://www.goodtoknow.co.uk/feeds/sex.rss'),
+                    (u'Easy Exercise', u'http://www.goodtoknow.co.uk/feeds/easyexercise.rss'),
+                    (u'Recipes', u'http://www.goodtoknow.co.uk/feeds/recipes.rss'),
+                    (u'Food Quick-tips', u'http://www.goodtoknow.co.uk/feeds/foodquicktips.rss'),
+                    ]

recipes/impulse_de.recipe

@@ -0,0 +1,32 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1305470859(BasicNewsRecipe):
+    title                 = u'Impulse.de'
+    language              = 'de'
+    __author__            = 'schuster'
+    oldest_article        =14
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    use_embedded_content  = False
+    cover_url = 'http://www.bvk.de/files/image/bilder/Logo%20Impulse.jpg'
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h4{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    img {min-width:300px; max-width:600px; min-height:300px; max-height:800px}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+	'''
+    def print_version(self, url):
+        return url.replace ('#utm_source=rss2&utm_medium=rss_feed&utm_campaign=/', '?mode=print')
+    remove_tags_bevor = [dict(name='h1', attrs={'class':'h2'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'artikelfuss'})]
+
+    feeds = [   (u'impulstest', u'http://www.impulse.de/rss/')]
+
+
+    remove_tags = [dict(attrs={'class':['navSeitenAlle', 'kommentieren', 'teaserheader', 'teasercontent', 'info', 'zwischenhead', 'kasten_artikel']}),
+                dict(id=['metaNav', 'impKopf', 'impTopNav', 'impSubNav', 'footerRahmen', 'gatrixx_marktinformationen', 'pager', 'weitere', 'socialmedia', 'rating_open']),
+                dict(span=['ratingtext', 'Gesamtranking', 'h3','']),
+	dict(rel=['canonical'])]
+

recipes/iprofesional.recipe

@@ -0,0 +1,79 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.iprofesional.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class iProfesional(BasicNewsRecipe):
+    title                 = 'iProfesional.com'
+    __author__            = 'Darko Miletic'
+    description           = 'Las ultimas noticias sobre profesionales'
+    publisher             = 'Emprendimientos Corporativos S.A.'
+    category              = 'news, IT, impuestos, negocios, politics, Argentina'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'es_AR'
+    remove_empty_feeds    = True
+    publication_type      = 'nesportal'
+    masthead_url          = 'http://www.iprofesional.com/img/logo-iprofesional.png'
+    extra_css             = """
+                               body{font-family: Arial,Helvetica,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                               .titulo-interior{font-family: Georgia,"Times New Roman",Times,serif}
+                               .autor-nota{font-size: small; font-weight: bold; font-style: italic; color: gray}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    keep_only_tags = [dict(attrs={'class':['fecha','interior-nota']})]
+
+    remove_tags = [
+                     dict(name=['meta','link','base','embed','object','iframe'])
+                    ,dict(attrs={'class':['menu-imprimir','guardarNota','IN-widget','fin','permalink']})
+                  ]
+    remove_attributes=['lang','xmlns:og','xmlns:fb']
+
+
+    feeds = [
+              (u'Ultimas noticias'  , u'http://feeds.feedburner.com/iprofesional-principales-noticias')
+             ,(u'Finanzas'          , u'http://feeds.feedburner.com/iprofesional-finanzas'            )
+             ,(u'Impuestos'         , u'http://feeds.feedburner.com/iprofesional-impuestos'           )
+             ,(u'Negocios'          , u'http://feeds.feedburner.com/iprofesional-economia'            )
+             ,(u'Comercio Exterior' , u'http://feeds.feedburner.com/iprofesional-comercio-exterior'   )
+             ,(u'Tecnologia'        , u'http://feeds.feedburner.com/iprofesional-tecnologia'          )
+             ,(u'Management'        , u'http://feeds.feedburner.com/iprofesional-managment'           )
+             ,(u'Marketing'         , u'http://feeds.feedburner.com/iprofesional-marketing'           )
+             ,(u'Legales'           , u'http://feeds.feedburner.com/iprofesional-legales'             )
+             ,(u'Autos'             , u'http://feeds.feedburner.com/iprofesional-autos'               )
+             ,(u'Vinos'             , u'http://feeds.feedburner.com/iprofesional-vinos-bodegas'       )
+            ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/kath_net.recipe

@@ -3,8 +3,9 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1295262156(BasicNewsRecipe):
     title          = u'kath.net'
     __author__     = 'Bobus'
+    description    = u'Katholische Nachrichten'
     oldest_article = 7
-    language = 'en'
+    language = 'de'
     max_articles_per_feed = 100
 
     feeds          = [(u'kath.net', u'http://www.kath.net/2005/xml/index.xml')]

recipes/liberty_times.recipe

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+# dug from http://www.mobileread.com/forums/showthread.php?p=1012294
+
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277443634(BasicNewsRecipe):
+	title = u'自由電子報'
+	oldest_article = 1
+	max_articles_per_feed = 100
+
+	feeds = [(u'焦點新聞', u'http://www.libertytimes.com.tw/rss/fo.xml'),
+                  (u'政治新聞', u'http://www.libertytimes.com.tw/rss/p.xml'),
+                  (u'生活新聞', u'http://www.libertytimes.com.tw/rss/life.xml'),
+                  (u'國際新聞', u'http://www.libertytimes.com.tw/rss/int.xml'),
+                  (u'自由廣場', u'http://www.libertytimes.com.tw/rss/o.xml'),
+                  (u'社會新聞', u'http://www.libertytimes.com.tw/rss/so.xml'),
+                  (u'體育新聞', u'http://www.libertytimes.com.tw/rss/sp.xml'),
+                  (u'財經焦點', u'http://www.libertytimes.com.tw/rss/e.xml'),
+                  (u'證券理財', u'http://www.libertytimes.com.tw/rss/stock.xml'),
+                  (u'影視焦點', u'http://www.libertytimes.com.tw/rss/show.xml'),
+                  (u'北部新聞', u'http://www.libertytimes.com.tw/rss/north.xml'),
+                  (u'中部新聞', u'http://www.libertytimes.com.tw/rss/center.xml'),
+                  (u'南部新聞', u'http://www.libertytimes.com.tw/rss/south.xml'),
+                  (u'大台北新聞', u'http://www.libertytimes.com.tw/rss/taipei.xml'),
+                  (u'藝術文化', u'http://www.libertytimes.com.tw/rss/art.xml'),
+                 ]
+	extra_css = '''span[class='insubject1'][id='newtitle'] {font-size:200%; font-weight:bold;}'''
+	__author__ = 'einstuerzende, updated by Eddie Lau'
+	__version__ = '1.1'
+	language = 'zh'
+	publisher = 'Liberty Times Group'
+	description = 'Liberty Times (Taiwan)'
+	category = 'News, Chinese, Taiwan'
+	remove_javascript = True
+	use_embedded_content = False
+	no_stylesheets = True
+	encoding = 'big5'
+	conversion_options = {'linearize_tables':True}
+	masthead_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif'
+	cover_url = 'http://www.libertytimes.com.tw/2008/images/img_auto/005/logo_new.gif'
+	keep_only_tags = [dict(name='td', attrs={'id':['newsContent']})]
+

recipes/max_planck.recipe

@@ -0,0 +1,22 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Max-Planck-Inst.'
+    __author__  = 'schuster'
+    remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}),
+                dict(id=['ie_clearing', 'col2', 'col2_content']),
+                dict(name=['script', 'noscript', 'style'])]
+    oldest_article = 30
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+
+    def print_version(self, url):
+        split_url = url.split("/")
+        print_url = 'http://www.mpg.de/print/' +  split_url[3]
+        return print_url
+
+    feeds          = [(u'Forschung', u'http://www.mpg.de/de/forschung.rss')]
+

recipes/mayra.recipe

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+mayra.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Mayra(BasicNewsRecipe):
+    title                 = u'Mayra'
+    language              = 'ro'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Traieste urban, cool, sexy'
+    publisher             = 'Mayra'
+    category              = 'Ziare,Stiri,Reviste'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    remove_javascript     = True
+    cover_url             = 'http://img.konkurs.ro/img/concursuri-cu-premii/147/14672_front.jpg'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                        dict(name='div', attrs={'id':'article_details'})
+                     ]
+
+    remove_tags = [
+                     dict(name='div', attrs={'id':'LikePluginPagelet'})
+                     , dict(name='p', attrs={'id':'tags'})
+                    , dict(name='span', attrs={'id':'tweet-button'})
+                  ]
+
+    remove_tags_after = [
+                             dict(name='div', attrs={'id':'LikePluginPagelet'})
+                        ]
+
+    feeds = [ (u'\u0218tiri', u'http://www.mayra.ro/rss') ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/mens_health.recipe

@@ -0,0 +1,10 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1305636254(BasicNewsRecipe):
+    title          = u'Mens Health (US)'
+    language = 'en'
+    __author__ = 'Anonymous'
+    oldest_article = 14
+    max_articles_per_feed = 100
+
+    feeds          = [(u'News', u'http://blogs.menshealth.com/health-headlines/feed')]

recipes/ming_pao.recipe

@@ -1,15 +1,18 @@
+# -*- coding: utf-8 -*-
 __license__   = 'GPL v3'
 __copyright__ = '2010-2011, Eddie Lau'
 
-# Users of Kindle 3 (with limited system-level CJK support)
+# Users of Kindle 3 with limited system-level CJK support
 # please replace the following "True" with "False".
 __MakePeriodical__ = True
-# Turn it to True if your device supports display of CJK titles
+# Turn below to true if your device supports display of CJK titles
 __UseChineseTitle__ = False
-
+# Trun below to true if you wish to use life.mingpao.com as the main article source
+__UseLife__ = True
 
 '''
 Change Log:
+2011/05/12: switch the main parse source to life.mingpao.com, which has more photos on the article pages
 2011/03/06: add new articles for finance section, also a new section "Columns"
 2011/02/28: rearrange the sections
             [Disabled until Kindle has better CJK support and can remember last (section,article) read in Sections & Articles
@@ -32,41 +35,43 @@ import os, datetime, re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 from contextlib import nested
 
+
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation
 
 class MPHKRecipe(BasicNewsRecipe):
-	title          = 'Ming Pao - Hong Kong'
-	oldest_article = 1
-	max_articles_per_feed = 100
-	__author__            = 'Eddie Lau'
-	description           = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
-	publisher             = 'MingPao'
-	category              = 'Chinese, News, Hong Kong'
-	remove_javascript = True
-	use_embedded_content   = False
-	no_stylesheets = True
-	language = 'zh'
-	encoding = 'Big5-HKSCS'
-	recursions = 0
-	conversion_options = {'linearize_tables':True}
-	timefmt = ''
-	extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
-	masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
-	keep_only_tags = [dict(name='h1'),
+    title          = 'Ming Pao - Hong Kong'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    __author__            = 'Eddie Lau'
+    description           = 'Hong Kong Chinese Newspaper (http://news.mingpao.com)'
+    publisher             = 'MingPao'
+    category              = 'Chinese, News, Hong Kong'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'zh'
+    encoding = 'Big5-HKSCS'
+    recursions = 0
+    conversion_options = {'linearize_tables':True}
+    timefmt = ''
+    extra_css = 'img {display: block; margin-left: auto; margin-right: auto; margin-top: 10px; margin-bottom: 10px;} font>b {font-size:200%; font-weight:bold;}'
+    masthead_url = 'http://news.mingpao.com/image/portals_top_logo_news.gif'
+    keep_only_tags = [dict(name='h1'),
                       dict(name='font', attrs={'style':['font-size:14pt; line-height:160%;']}), # for entertainment page title
                       dict(name='font', attrs={'color':['AA0000']}), # for column articles title
                       dict(attrs={'id':['newscontent']}), # entertainment and column page content
                       dict(attrs={'id':['newscontent01','newscontent02']}),
-                      dict(attrs={'class':['photo']})
+                      dict(attrs={'class':['photo']}),
+                      dict(name='img', attrs={'width':['180'], 'alt':['按圖放大']}) # images for source from life.mingpao.com
                       ]
-	remove_tags = [dict(name='style'),
-    			   dict(attrs={'id':['newscontent135']}),  # for the finance page
-    			   dict(name='table')]  # for content fetched from life.mingpao.com
-	remove_attributes = ['width']
-	preprocess_regexps = [
+    remove_tags = [dict(name='style'),
+                   dict(attrs={'id':['newscontent135']}),  # for the finance page from mpfinance.com
+                   dict(name='table')]  # for content fetched from life.mingpao.com
+    remove_attributes = ['width']
+    preprocess_regexps = [
                           (re.compile(r'<h5>', re.DOTALL|re.IGNORECASE),
                           lambda match: '<h1>'),
                           (re.compile(r'</h5>', re.DOTALL|re.IGNORECASE),
@@ -80,10 +85,10 @@ class MPHKRecipe(BasicNewsRecipe):
                           lambda match: "</b>")
                          ]
 
-	def image_url_processor(cls, baseurl, url):
-		# trick: break the url at the first occurance of digit, add an additional
-		# '_' at the front
-		# not working, may need to move this to preprocess_html() method
+    def image_url_processor(cls, baseurl, url):
+        # trick: break the url at the first occurance of digit, add an additional
+        # '_' at the front
+        # not working, may need to move this to preprocess_html() method
 #        minIdx = 10000
 #        i0 = url.find('0')
 #        if i0 >= 0 and i0 < minIdx:
@@ -115,314 +120,357 @@ class MPHKRecipe(BasicNewsRecipe):
 #        i9 = url.find('9')
 #        if i9 >= 0 and i9 < minIdx:
 #           minIdx = i9
-		return url
+        return url
 
-	def get_dtlocal(self):
-		dt_utc = datetime.datetime.utcnow()
-		# convert UTC to local hk time - at around HKT 6.00am, all news are available
-		dt_local = dt_utc - datetime.timedelta(-2.0/24)
-		return dt_local
+    def get_dtlocal(self):
+        dt_utc = datetime.datetime.utcnow()
+        # convert UTC to local hk time - at around HKT 6.00am, all news are available
+        dt_local = dt_utc - datetime.timedelta(-2.0/24)
+        return dt_local
 
-	def get_fetchdate(self):
-		return self.get_dtlocal().strftime("%Y%m%d")
+    def get_fetchdate(self):
+        return self.get_dtlocal().strftime("%Y%m%d")
 
-	def get_fetchformatteddate(self):
-		return self.get_dtlocal().strftime("%Y-%m-%d")
+    def get_fetchformatteddate(self):
+        return self.get_dtlocal().strftime("%Y-%m-%d")
 
-	def get_fetchday(self):
-		# convert UTC to local hk time - at around HKT 6.00am, all news are available
-		return self.get_dtlocal().strftime("%d")
+    def get_fetchday(self):
+        # dt_utc = datetime.datetime.utcnow()
+        # convert UTC to local hk time - at around HKT 6.00am, all news are available
+        # dt_local = dt_utc - datetime.timedelta(-2.0/24)
+        return self.get_dtlocal().strftime("%d")
 
-	def get_cover_url(self):
-		cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
-		br = BasicNewsRecipe.get_browser()
-		try:
-			br.open(cover)
-		except:
-			cover = None
-		return cover
+    def get_cover_url(self):
+        cover = 'http://news.mingpao.com/' + self.get_fetchdate() + '/' + self.get_fetchdate() + '_' + self.get_fetchday() + 'gacov.jpg'
+        br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover)
+        except:
+            cover = None
+        return cover
 
-	def parse_index(self):
-		feeds = []
-		dateStr = self.get_fetchdate()
+    def parse_index(self):
+        feeds = []
+        dateStr = self.get_fetchdate()
 
-		for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
-		                   (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
-		                   (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
-			articles = self.parse_section(url)
-			if articles:
-				feeds.append((title, articles))
+        if __UseLife__:
+            for title, url, keystr in [(u'\u8981\u805e Headline', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalga', 'nal'),
+                                       (u'\u6e2f\u805e Local', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgb', 'nal'),
+                                       (u'\u6559\u80b2 Education', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalgf', 'nal'),
+                                       (u'\u793e\u8a55/\u7b46\u9663 Editorial', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr', 'nal'),
+                                       (u'\u8ad6\u58c7 Forum', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalfa', 'nal'),
+                                       (u'\u4e2d\u570b China', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalca', 'nal'),
+                                       (u'\u570b\u969b World', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalta', 'nal'),
+                                       (u'\u7d93\u6fdf Finance', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea', 'nal'),
+                                       (u'\u9ad4\u80b2 Sport', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalsp', 'nal'),
+                                       (u'\u5f71\u8996 Film/TV', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalma', 'nal'),
+                                       (u'\u5c08\u6b04 Columns', 'http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn', 'ncl')]:
+                articles = self.parse_section2(url, keystr)
+                if articles:
+                    feeds.append((title, articles))
 
-		# special- editorial
-		ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
-		if ed_articles:
-			feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
+            for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
+                               (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
+                articles = self.parse_section(url)
+                if articles:
+                    feeds.append((title, articles))
+        else:
+            for title, url in [(u'\u8981\u805e Headline', 'http://news.mingpao.com/' + dateStr + '/gaindex.htm'),
+                               (u'\u6e2f\u805e Local', 'http://news.mingpao.com/' + dateStr + '/gbindex.htm'),
+                               (u'\u6559\u80b2 Education', 'http://news.mingpao.com/' + dateStr + '/gfindex.htm')]:
+                articles = self.parse_section(url)
+                if articles:
+                    feeds.append((title, articles))
 
-		for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
-                           (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
-                           (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
-			articles = self.parse_section(url)
-			if articles:
-				feeds.append((title, articles))
+            # special- editorial
+            ed_articles = self.parse_ed_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=nalmr')
+            if ed_articles:
+                feeds.append((u'\u793e\u8a55/\u7b46\u9663 Editorial', ed_articles))
 
-		# special - finance
-		#fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
-		fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
-		if fin_articles:
-			feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
+            for title, url in [(u'\u8ad6\u58c7 Forum', 'http://news.mingpao.com/' + dateStr + '/faindex.htm'),
+                               (u'\u4e2d\u570b China', 'http://news.mingpao.com/' + dateStr + '/caindex.htm'),
+                               (u'\u570b\u969b World', 'http://news.mingpao.com/' + dateStr + '/taindex.htm')]:
+                articles = self.parse_section(url)
+                if articles:
+                    feeds.append((title, articles))
 
-		for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
-                           (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
-			articles = self.parse_section(url)
-			if articles:
-				feeds.append((title, articles))
+            # special - finance
+            #fin_articles = self.parse_fin_section('http://www.mpfinance.com/htm/Finance/' + dateStr + '/News/ea,eb,ecindex.htm')
+            fin_articles = self.parse_fin_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr + '&Category=nalea')
+            if fin_articles:
+                feeds.append((u'\u7d93\u6fdf Finance', fin_articles))
 
-		# special - entertainment
-		ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
-		if ent_articles:
-			feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
+            for title, url in [('Tech News', 'http://news.mingpao.com/' + dateStr + '/naindex.htm'),
+                               (u'\u9ad4\u80b2 Sport', 'http://news.mingpao.com/' + dateStr + '/spindex.htm')]:
+                articles = self.parse_section(url)
+                if articles:
+                    feeds.append((title, articles))
 
-		for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
-                           (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
-			articles = self.parse_section(url)
-			if articles:
-				feeds.append((title, articles))
+            # special - entertainment
+            ent_articles = self.parse_ent_section('http://ol.mingpao.com/cfm/star1.cfm')
+            if ent_articles:
+                feeds.append((u'\u5f71\u8996 Film/TV', ent_articles))
+
+            for title, url in [(u'\u526f\u520a Supplement', 'http://news.mingpao.com/' + dateStr + '/jaindex.htm'),
+                               (u'\u82f1\u6587 English', 'http://news.mingpao.com/' + dateStr + '/emindex.htm')]:
+                articles = self.parse_section(url)
+                if articles:
+                    feeds.append((title, articles))
 
 
-		# special- columns
-		col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
-		if col_articles:
-			feeds.append((u'\u5c08\u6b04 Columns', col_articles))
+            # special- columns
+            col_articles = self.parse_col_section('http://life.mingpao.com/cfm/dailynews2.cfm?Issue=' + dateStr +'&Category=ncolumn')
+            if col_articles:
+                feeds.append((u'\u5c08\u6b04 Columns', col_articles))
 
-		return feeds
+        return feeds
 
-	def parse_section(self, url):
-		dateStr = self.get_fetchdate()
-		soup = self.index_to_soup(url)
-		divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']})
-		current_articles = []
-		included_urls = []
-		divs.reverse()
-		for i in divs:
-			a = i.find('a', href = True)
-			title = self.tag_to_string(a)
-			url = a.get('href', False)
-			url = 'http://news.mingpao.com/' + dateStr + '/' +url
-			if url not in included_urls and url.rfind('Redirect') == -1:
-				current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
-				included_urls.append(url)
-		current_articles.reverse()
-		return current_articles
+    # parse from news.mingpao.com
+    def parse_section(self, url):
+        dateStr = self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        divs = soup.findAll(attrs={'class': ['bullet','bullet_grey']})
+        current_articles = []
+        included_urls = []
+        divs.reverse()
+        for i in divs:
+            a = i.find('a', href = True)
+            title = self.tag_to_string(a)
+            url = a.get('href', False)
+            url = 'http://news.mingpao.com/' + dateStr + '/' +url
+            if url not in included_urls and url.rfind('Redirect') == -1:
+                current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles
 
-	def parse_ed_section(self, url):
-		self.get_fetchdate()
-		soup = self.index_to_soup(url)
-		a = soup.findAll('a', href=True)
-		a.reverse()
-		current_articles = []
-		included_urls = []
-		for i in a:
-			title = self.tag_to_string(i)
-			url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
-			if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1):
-				current_articles.append({'title': title, 'url': url, 'description': ''})
-				included_urls.append(url)
-		current_articles.reverse()
-		return current_articles
+    # parse from life.mingpao.com
+    def parse_section2(self, url, keystr):
+        self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href=True)
+        a.reverse()
+        current_articles = []
+        included_urls = []
+        for i in a:
+            title = self.tag_to_string(i)
+            url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
+            if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind(keystr) == -1):
+                current_articles.append({'title': title, 'url': url, 'description': ''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles
 
-	def parse_fin_section(self, url):
-		self.get_fetchdate()
-		soup = self.index_to_soup(url)
-		a = soup.findAll('a', href= True)
-		current_articles = []
-		included_urls = []
-		for i in a:
-			#url = 'http://www.mpfinance.com/cfm/' + i.get('href', False)
-			url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
-			#if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1:
-			if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1):
-				title = self.tag_to_string(i)
-				current_articles.append({'title': title, 'url': url, 'description':''})
-				included_urls.append(url)
-		return current_articles
+    def parse_ed_section(self, url):
+        self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href=True)
+        a.reverse()
+        current_articles = []
+        included_urls = []
+        for i in a:
+            title = self.tag_to_string(i)
+            url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
+            if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('nal') == -1):
+                current_articles.append({'title': title, 'url': url, 'description': ''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles
 
-	def parse_ent_section(self, url):
-		self.get_fetchdate()
-		soup = self.index_to_soup(url)
-		a = soup.findAll('a', href=True)
-		a.reverse()
-		current_articles = []
-		included_urls = []
-		for i in a:
-			title = self.tag_to_string(i)
-			url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
-			if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1):
-				current_articles.append({'title': title, 'url': url, 'description': ''})
-				included_urls.append(url)
-		current_articles.reverse()
-		return current_articles
+    def parse_fin_section(self, url):
+        self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href= True)
+        current_articles = []
+        included_urls = []
+        for i in a:
+            #url = 'http://www.mpfinance.com/cfm/' + i.get('href', False)
+            url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
+            #if url not in included_urls and not url.rfind(dateStr) == -1 and url.rfind('index') == -1:
+            if url not in included_urls and (not url.rfind('txt') == -1) and (not url.rfind('nal') == -1):
+                title = self.tag_to_string(i)
+                current_articles.append({'title': title, 'url': url, 'description':''})
+                included_urls.append(url)
+        return current_articles
 
-	def parse_col_section(self, url):
-		self.get_fetchdate()
-		soup = self.index_to_soup(url)
-		a = soup.findAll('a', href=True)
-		a.reverse()
-		current_articles = []
-		included_urls = []
-		for i in a:
-			title = self.tag_to_string(i)
-			url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
-			if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1):
-				current_articles.append({'title': title, 'url': url, 'description': ''})
-				included_urls.append(url)
-		current_articles.reverse()
-		return current_articles
+    def parse_ent_section(self, url):
+        self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href=True)
+        a.reverse()
+        current_articles = []
+        included_urls = []
+        for i in a:
+            title = self.tag_to_string(i)
+            url = 'http://ol.mingpao.com/cfm/' + i.get('href', False)
+            if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('star') == -1):
+                current_articles.append({'title': title, 'url': url, 'description': ''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles
 
-	def preprocess_html(self, soup):
-		for item in soup.findAll(style=True):
-			del item['style']
-		for item in soup.findAll(style=True):
-			del item['width']
-		for item in soup.findAll(stype=True):
-			del item['absmiddle']
-		return soup
+    def parse_col_section(self, url):
+        self.get_fetchdate()
+        soup = self.index_to_soup(url)
+        a = soup.findAll('a', href=True)
+        a.reverse()
+        current_articles = []
+        included_urls = []
+        for i in a:
+            title = self.tag_to_string(i)
+            url = 'http://life.mingpao.com/cfm/' + i.get('href', False)
+            if (url not in included_urls) and (not url.rfind('.txt') == -1) and (not url.rfind('ncl') == -1):
+                current_articles.append({'title': title, 'url': url, 'description': ''})
+                included_urls.append(url)
+        current_articles.reverse()
+        return current_articles
 
-	def create_opf(self, feeds, dir=None):
-		if dir is None:
-			dir = self.output_dir
-		if __UseChineseTitle__ == True:
-			title = u'\u660e\u5831 (\u9999\u6e2f)'
-		else:
-			title = self.short_title()
-		# if not generating a periodical, force date to apply in title
-		if __MakePeriodical__ == False:
-			title = title + ' ' + self.get_fetchformatteddate()
-		if True:
-			mi = MetaInformation(title, [self.publisher])
-			mi.publisher = self.publisher
-			mi.author_sort = self.publisher
-			if __MakePeriodical__ == True:
-				mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
-			else:
-				mi.publication_type = self.publication_type+':'+self.short_title()
-			#mi.timestamp = nowf()
-			mi.timestamp = self.get_dtlocal()
-			mi.comments = self.description
-			if not isinstance(mi.comments, unicode):
-				mi.comments = mi.comments.decode('utf-8', 'replace')
-			#mi.pubdate = nowf()
-			mi.pubdate = self.get_dtlocal()
-			opf_path = os.path.join(dir, 'index.opf')
-			ncx_path = os.path.join(dir, 'index.ncx')
-			opf = OPFCreator(dir, mi)
-			# Add mastheadImage entry to <guide> section
-			mp = getattr(self, 'masthead_path', None)
-			if mp is not None and os.access(mp, os.R_OK):
-				from calibre.ebooks.metadata.opf2 import Guide
-				ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu())
-				ref.type = 'masthead'
-				ref.title = 'Masthead Image'
-				opf.guide.append(ref)
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(style=True):
+            del item['width']
+        for item in soup.findAll(stype=True):
+            del item['absmiddle']
+        return soup
 
-			manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
-			manifest.append(os.path.join(dir, 'index.html'))
-			manifest.append(os.path.join(dir, 'index.ncx'))
+    def create_opf(self, feeds, dir=None):
+        if dir is None:
+            dir = self.output_dir
+        if __UseChineseTitle__ == True:
+            title = u'\u660e\u5831 (\u9999\u6e2f)'
+        else:
+            title = self.short_title()
+        # if not generating a periodical, force date to apply in title
+        if __MakePeriodical__ == False:
+            title = title + ' ' + self.get_fetchformatteddate()
+        if True:
+            mi = MetaInformation(title, [self.publisher])
+            mi.publisher = self.publisher
+            mi.author_sort = self.publisher
+            if __MakePeriodical__ == True:
+                mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
+            else:
+                mi.publication_type = self.publication_type+':'+self.short_title()
+            #mi.timestamp = nowf()
+            mi.timestamp = self.get_dtlocal()
+            mi.comments = self.description
+            if not isinstance(mi.comments, unicode):
+                mi.comments = mi.comments.decode('utf-8', 'replace')
+            #mi.pubdate = nowf()
+            mi.pubdate = self.get_dtlocal()
+            opf_path = os.path.join(dir, 'index.opf')
+            ncx_path = os.path.join(dir, 'index.ncx')
+            opf = OPFCreator(dir, mi)
+            # Add mastheadImage entry to <guide> section
+            mp = getattr(self, 'masthead_path', None)
+            if mp is not None and os.access(mp, os.R_OK):
+                from calibre.ebooks.metadata.opf2 import Guide
+                ref = Guide.Reference(os.path.basename(self.masthead_path), os.getcwdu())
+                ref.type = 'masthead'
+                ref.title = 'Masthead Image'
+                opf.guide.append(ref)
 
-			# Get cover
-			cpath = getattr(self, 'cover_path', None)
-			if cpath is None:
-				pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
-				if self.default_cover(pf):
-					cpath =  pf.name
-			if cpath is not None and os.access(cpath, os.R_OK):
-				opf.cover = cpath
-				manifest.append(cpath)
+            manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
+            manifest.append(os.path.join(dir, 'index.html'))
+            manifest.append(os.path.join(dir, 'index.ncx'))
 
-			# Get masthead
-			mpath = getattr(self, 'masthead_path', None)
-			if mpath is not None and os.access(mpath, os.R_OK):
-				manifest.append(mpath)
+            # Get cover
+            cpath = getattr(self, 'cover_path', None)
+            if cpath is None:
+                pf = open(os.path.join(dir, 'cover.jpg'), 'wb')
+                if self.default_cover(pf):
+                    cpath =  pf.name
+            if cpath is not None and os.access(cpath, os.R_OK):
+                opf.cover = cpath
+                manifest.append(cpath)
 
-			opf.create_manifest_from_files_in(manifest)
-			for mani in opf.manifest:
-				if mani.path.endswith('.ncx'):
-					mani.id = 'ncx'
-				if mani.path.endswith('mastheadImage.jpg'):
-					mani.id = 'masthead-image'
-			entries = ['index.html']
-			toc = TOC(base_path=dir)
-			self.play_order_counter = 0
-			self.play_order_map = {}
+            # Get masthead
+            mpath = getattr(self, 'masthead_path', None)
+            if mpath is not None and os.access(mpath, os.R_OK):
+                manifest.append(mpath)
 
-		def feed_index(num, parent):
-			f = feeds[num]
-			for j, a in enumerate(f):
-				if getattr(a, 'downloaded', False):
-					adir = 'feed_%d/article_%d/'%(num, j)
-					auth = a.author
-					if not auth:
-						auth = None
-					desc = a.text_summary
-					if not desc:
-						desc = None
-					else:
-						desc = self.description_limiter(desc)
-					entries.append('%sindex.html'%adir)
-					po = self.play_order_map.get(entries[-1], None)
-					if po is None:
-						self.play_order_counter += 1
-						po = self.play_order_counter
-					parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'),
+            opf.create_manifest_from_files_in(manifest)
+            for mani in opf.manifest:
+                if mani.path.endswith('.ncx'):
+                    mani.id = 'ncx'
+                if mani.path.endswith('mastheadImage.jpg'):
+                    mani.id = 'masthead-image'
+            entries = ['index.html']
+            toc = TOC(base_path=dir)
+            self.play_order_counter = 0
+            self.play_order_map = {}
+
+        def feed_index(num, parent):
+            f = feeds[num]
+            for j, a in enumerate(f):
+                if getattr(a, 'downloaded', False):
+                    adir = 'feed_%d/article_%d/'%(num, j)
+                    auth = a.author
+                    if not auth:
+                        auth = None
+                    desc = a.text_summary
+                    if not desc:
+                        desc = None
+                    else:
+                        desc = self.description_limiter(desc)
+                    entries.append('%sindex.html'%adir)
+                    po = self.play_order_map.get(entries[-1], None)
+                    if po is None:
+                        self.play_order_counter += 1
+                        po = self.play_order_counter
+                    parent.add_item('%sindex.html'%adir, None, a.title if a.title else _('Untitled Article'),
                                     play_order=po, author=auth, description=desc)
-					last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep))
-					for sp in a.sub_pages:
-						prefix = os.path.commonprefix([opf_path, sp])
-						relp = sp[len(prefix):]
-						entries.append(relp.replace(os.sep, '/'))
-						last = sp
+                    last = os.path.join(self.output_dir, ('%sindex.html'%adir).replace('/', os.sep))
+                    for sp in a.sub_pages:
+                        prefix = os.path.commonprefix([opf_path, sp])
+                        relp = sp[len(prefix):]
+                        entries.append(relp.replace(os.sep, '/'))
+                        last = sp
 
-					if os.path.exists(last):
-						with open(last, 'rb') as fi:
-							src = fi.read().decode('utf-8')
-						soup = BeautifulSoup(src)
-						body = soup.find('body')
-						if body is not None:
-							prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last))))
-							templ = self.navbar.generate(True, num, j, len(f),
+                    if os.path.exists(last):
+                        with open(last, 'rb') as fi:
+                            src = fi.read().decode('utf-8')
+                        soup = BeautifulSoup(src)
+                        body = soup.find('body')
+                        if body is not None:
+                            prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last))))
+                            templ = self.navbar.generate(True, num, j, len(f),
                                             not self.has_single_feed,
                                             a.orig_url, self.publisher, prefix=prefix,
                                             center=self.center_navbar)
-							elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
-							body.insert(len(body.contents), elem)
-							with open(last, 'wb') as fi:
-								fi.write(unicode(soup).encode('utf-8'))
-		if len(feeds) == 0:
-			raise Exception('All feeds are empty, aborting.')
+                            elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
+                            body.insert(len(body.contents), elem)
+                            with open(last, 'wb') as fi:
+                                fi.write(unicode(soup).encode('utf-8'))
+        if len(feeds) == 0:
+            raise Exception('All feeds are empty, aborting.')
 
-		if len(feeds) > 1:
-			for i, f in enumerate(feeds):
-				entries.append('feed_%d/index.html'%i)
-				po = self.play_order_map.get(entries[-1], None)
-				if po is None:
-					self.play_order_counter += 1
-					po = self.play_order_counter
-				auth = getattr(f, 'author', None)
-				if not auth:
-					auth = None
-				desc = getattr(f, 'description', None)
-				if not desc:
-					desc = None
-				feed_index(i, toc.add_item('feed_%d/index.html'%i, None,
+        if len(feeds) > 1:
+            for i, f in enumerate(feeds):
+                entries.append('feed_%d/index.html'%i)
+                po = self.play_order_map.get(entries[-1], None)
+                if po is None:
+                    self.play_order_counter += 1
+                    po = self.play_order_counter
+                auth = getattr(f, 'author', None)
+                if not auth:
+                    auth = None
+                desc = getattr(f, 'description', None)
+                if not desc:
+                    desc = None
+                feed_index(i, toc.add_item('feed_%d/index.html'%i, None,
                            f.title, play_order=po, description=desc, author=auth))
 
-		else:
-			entries.append('feed_%d/index.html'%0)
-			feed_index(0, toc)
+        else:
+            entries.append('feed_%d/index.html'%0)
+            feed_index(0, toc)
 
-		for i, p in enumerate(entries):
-			entries[i] = os.path.join(dir, p.replace('/', os.sep))
-		opf.create_spine(entries)
-		opf.set_toc(toc)
+        for i, p in enumerate(entries):
+            entries[i] = os.path.join(dir, p.replace('/', os.sep))
+        opf.create_spine(entries)
+        opf.set_toc(toc)
+
+        with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
+            opf.render(opf_file, ncx_file)
 
-		with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
-			opf.render(opf_file, ncx_file)

recipes/moldovaazi.recipe

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+azi.md
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class MoldovaAzi(BasicNewsRecipe):
+    title                 = u'Moldova Azi'
+    language              = 'ro'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Moldova pe internet'
+    publisher             = 'Moldova Azi'
+    category              = 'Ziare,Stiri,Moldova'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    remove_javascript     = True
+    cover_url             = 'http://www.azi.md/images/logo.gif'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [  dict(name='div', attrs={'id':'in'})
+                     ]
+
+
+    remove_tags = [
+                     dict(name='div', attrs={'class':'in-more-stories'})
+                  ]
+
+    remove_tags_after = [
+                              dict(name='div', attrs={'id':'comment_wrapper'})
+                            , dict(name='div', attrs={'class':'box-title4'})
+                        ]
+
+    feeds = [ (u'\u0218tiri', u'http://www.azi.md/ro/feeds/0/rss201') ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/natgeo.recipe

@@ -0,0 +1,71 @@
+#!/usr/bin/env  python
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+__copyright__ = '2011, gagsays <gagsays at gmail dot com>'
+'''
+nationalgeographic.com
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+class NatGeo(BasicNewsRecipe):
+    title          = u'National Geographic'
+    description = 'Daily news articles from The National Geographic'
+    language = 'en'
+    oldest_article = 20
+    max_articles_per_feed = 25
+    encoding              = 'utf8'
+    publisher              = 'nationalgeographic.com'
+    category               = 'science, nat geo'
+    __author__           = 'gagsays'
+    masthead_url        = 'http://s.ngeo.com/wpf/sites/themes/global/i/presentation/ng_logo_small.png'
+    description           = 'Inspiring people to care about the planet since 1888'
+    timefmt = ' [%a, %d %b, %Y]'
+    no_stylesheets        = True
+    use_embedded_content  = False
+
+    extra_css = '''
+                body {color: #000000;font-size: medium;}
+                h1 {color: #222222; font-size: large; font-weight:lighter; text-decoration:none; text-align: center;font-family:Georgia,Times New Roman,Times,serif;}
+          h2 {color: #454545; font-size: small; font-weight:lighter; text-decoration:none; text-align: justify; font-style:italic;font-family :Georgia,Times New Roman,Times,serif;}
+                h3 {color: #555555; font-size: small; font-style:italic; margin-top: 10px;}
+                img{margin-bottom: 0.25em;display:block;margin-left: auto;margin-right: auto;}
+                a:link,a,.a,href {text-decoration: none;color: #000000;}
+                .caption{color: #000000;font-size: xx-small;text-align: justify;font-weight:normal;}
+                .credit{color: #555555;font-size: xx-small;text-align: left;font-weight:lighter;}
+                p.author,p.publication{color: #000000;font-size: xx-small;text-align: left;display:inline;}
+                p.publication_time{color: #000000;font-size: xx-small;text-align: right;text-decoration: underline;}
+                p {margin-bottom: 0;}
+                p + p {text-indent: 1.5em;margin-top: 0;}
+                .hidden{display:none;}
+                #page_head{text-transform:uppercase;}
+               '''
+
+    def parse_feeds (self):
+      feeds = BasicNewsRecipe.parse_feeds(self)
+      for feed in feeds:
+        for article in feed.articles[:]:
+          if 'Presented' in article.title or 'Pictures' in article.title:
+            feed.articles.remove(article)
+      return feeds
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    remove_tags_before = dict(id='page_head')
+    keep_only_tags = [
+         dict(name='div',attrs={'id':['page_head','content_mainA']})
+    ]
+    remove_tags_after = [
+         dict(name='div',attrs={'class':['article_text','promo_collection']})
+    ]
+    remove_tags    = [
+                               dict(name='div', attrs={'class':['aside','primary full_width']})
+                               ,dict(name='div', attrs={'id':['header_search','navigation_mainB_wrap']})
+                                ]
+    feeds = [
+                    (u'Daily News', u'http://feeds.nationalgeographic.com/ng/News/News_Main')
+                    ]
+

recipes/newsmoldova.recipe

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+newsmoldova.md
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class NewsMoldova(BasicNewsRecipe):
+    title                 = u'Agen\u0163ia de \u015ftiri Moldova'
+    language              = 'ro'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'Agen\u0163ia de \u015ftiri Moldova'
+    publisher             = 'Moldova'
+    category              = 'Ziare,Stiri,Moldova'
+    oldest_article        = 5
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    remove_javascript     = True
+    cover_url             = 'http://www.newsmoldova.md/i/logo_top_md.gif'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+                            ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [  dict(name='div', attrs={'class':'main-article-index article'})
+                     ]
+
+
+    remove_tags = [
+                      dict(name='div', attrs={'id':'actions'})
+                     , dict(name='li', attrs={'class':'invisible'})
+                  ]
+
+    remove_tags_after = [
+                             dict(name='div', attrs={'id':'actions'})
+                        ]
+
+    feeds = [ (u'\u0218tiri', u'http://newsmoldova.md/export/rss2/archive/index.xml') ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/newsweek.recipe

@@ -11,6 +11,20 @@ class Newsweek(BasicNewsRecipe):
 
     BASE_URL = 'http://www.newsweek.com'
 
+    topics = {
+        'Culture' : '/tag/culture.html',
+        'Business' : '/tag/business.html',
+        'Society' : '/tag/society.html',
+        'Science' : '/tag/science.html',
+        'Education' : '/tag/education.html',
+        'Politics' : '/tag/politics.html',
+        'Health' : '/tag/health.html',
+        'World' : '/tag/world.html',
+        'Nation' : '/tag/nation.html',
+        'Technology' : '/tag/technology.html',
+        'Game Changers' : '/tag/game-changers.html',
+    }
+
     keep_only_tags = dict(name='article', attrs={'class':'article-text'})
     remove_tags = [dict(attrs={'data-dartad':True})]
     remove_attributes = ['property']
@@ -21,14 +35,10 @@ class Newsweek(BasicNewsRecipe):
         return soup
 
     def newsweek_sections(self):
-        return [
-                ('Nation', 'http://www.newsweek.com/tag/nation.html'),
-                ('Society', 'http://www.newsweek.com/tag/society.html'),
-                ('Culture', 'http://www.newsweek.com/tag/culture.html'),
-                ('World', 'http://www.newsweek.com/tag/world.html'),
-                ('Politics', 'http://www.newsweek.com/tag/politics.html'),
-                ('Business', 'http://www.newsweek.com/tag/business.html'),
-        ]
+        for topic_name, topic_url in self.topics.iteritems():
+            yield (topic_name,
+                    self.BASE_URL+topic_url)
+
 
     def newsweek_parse_section_page(self, soup):
         for article in soup.findAll('article', about=True,

recipes/ngz.recipe

@@ -0,0 +1,29 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'NGZ-online'
+    __author__  = 'schuster'
+    remove_tags_before = dict(id='bu')
+    remove_tags_after  = dict(id='noblock')
+    remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool', 'nextArticleLink clearfix', 'liketext']}),
+                dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index', 'Verlinken', 'vorheriger', 'LESERKOMMENTARE', 'bei facebook', 'bei twitter', 'Schreiben Sie jetzt Ihre Meinung:', 'Thema', 'Ihr Beitrag', 'Ihr Name', 'Ich möchte über weitere Lesermeinungen zu diesem Artikel per E-Mail informiert werden.', 'banneroben', 'bannerrechts', 'inserieren', 'stellen', 'auto', 'immobilien', 'kleinanzeige', 'tiere', 'ferienwohnung', 'NGZ Card', 'Mediengruppe RP', 'Werben', 'Newsletter', 'Wetter', 'RSS', 'Abo', 'Anzeigen', 'Redaktion', 'Schulprojekte', 'Gast', 'Mein NGZ', 'Nachrichten', 'Sport', 'Wirtschaft', 'Stadt-Infos', 'Bilderserien', 'Bookmarken', 'del.icio.us', 'Mister Wong', 'YiGG', 'Webnews', 'Shortnews', 'Twitter', 'Newsider', 'Facebook', 'StudiVZ/MeinVZ', 'Versenden', 'Drucken']),
+                dict(name=['script', 'noscript', 'style'])]
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    cover_url = 'http://www.rhein-kreis-neuss-macht-sport.de/sport/includes/bilder/ngz_logo.jpg'
+
+    def print_version(self, url):
+          return url + '?ot=de.circit.rpo.PopupPageLayout.ot'
+    feeds          = [
+(u'Grevenbroich', u'http://www.ngz-online.de/app/feed/rss/grevenbroich'),
+(u'Kreis Neuss', u'http://www.ngz-online.de/app/feed/rss/rheinkreisneuss'),
+(u'Dormagen', u'http://www.ngz-online.de/app/feed/rss/dormagen'),
+(u'J\xfcchen', u'http://www.ngz-online.de/app/feed/rss/juechen'),
+(u'Rommerskirchen', u'http://www.ngz-online.de/app/feed/rss/rommerskirchen')
+
+]
+

recipes/pro_physik.recipe

@@ -0,0 +1,22 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Pro Physik'
+    __author__  = 'schuster'
+    oldest_article = 4
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+    cover_url = 'http://www.pro-physik.de/Phy/images/site/prophysik_logo1.jpg'
+
+
+    def print_version(self, url):
+        return url.replace('leadArticle.do', 'print.do')
+
+
+    feeds          = [(u'Hightech', u'http://www.pro-physik.de/Phy/hightechfeed.xml'),
+                          (u'Forschung', u'http://www.pro-physik.de/Phy/forschungfeed.xml'),
+                          (u'Magazin', u'http://www.pro-physik.de/Phy/magazinfeed.xml')]
+

recipes/readers_digest.recipe

@@ -3,7 +3,6 @@ __license__   = 'GPL v3'
 '''
 '''
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.web.feeds import Feed
 
 
 class ReadersDigest(BasicNewsRecipe):
@@ -38,151 +37,20 @@ class ReadersDigest(BasicNewsRecipe):
                         '''
 
 
-    remove_tags = [
-        dict(name='h4', attrs={'class':'close'}),
-        dict(name='div', attrs={'class':'fromLine'}),
-        dict(name='img', attrs={'class':'colorTag'}),
-        dict(name='div', attrs={'id':'sponsorArticleHeader'}),
-        dict(name='div', attrs={'class':'horizontalAd'}),
-        dict(name='div', attrs={'id':'imageCounterLeft'}),
-        dict(name='div', attrs={'id':'commentsPrint'})
-        ]
-
-
     feeds = [
-            ('New in RD', 'http://feeds.rd.com/ReadersDigest'),
-            ('Jokes', 'http://feeds.rd.com/ReadersDigestJokes'),
-            ('Cartoons', 'http://feeds.rd.com/ReadersDigestCartoons'),
-            ('Blogs','http://feeds.rd.com/ReadersDigestBlogs')
+            ('Food', 'http://www.rd.com/food/feed'),
+            ('Health', 'http://www.rd.com/health/feed'),
+            ('Home', 'http://www.rd.com/home/feed'),
+            ('Family', 'http://www.rd.com/family/feed'),
+            ('Money', 'http://www.rd.com/money/feed'),
+            ('Travel', 'http://www.rd.com/travel/feed'),
         ]
 
     cover_url = 'http://www.rd.com/images/logo-main-rd.gif'
 
-
-
-#-------------------------------------------------------------------------------------------------
-
-    def print_version(self, url):
-
-        # Get the identity number of the current article and append it to the root print URL
-
-        if url.find('/article') > 0:
-            ident = url[url.find('/article')+8:url.find('.html?')-4]
-            url = 'http://www.rd.com/content/printContent.do?contentId=' + ident
-
-        elif url.find('/post') > 0:
-
-            # in this case, have to get the page itself to derive the Print page.
-            soup = self.index_to_soup(url)
-            newsoup = soup.find('ul',attrs={'class':'printBlock'})
-            url = 'http://www.rd.com' + newsoup('a')[0]['href']
-            url = url[0:url.find('&Keep')]
-
-        return url
-
-#-------------------------------------------------------------------------------------------------
-
-    def parse_index(self):
-
-        pages = [
-                ('Your America','http://www.rd.com/your-america-inspiring-people-and-stories', 'channelLeftContainer',{'class':'moreLeft'}),
-                # useless recipes ('Living Healthy','http://www.rd.com/living-healthy', 'channelLeftContainer',{'class':'moreLeft'}),
-                ('Advice and Know-How','http://www.rd.com/advice-and-know-how', 'channelLeftContainer',{'class':'moreLeft'})
-
+    keep_only_tags = dict(id='main-content')
+    remove_tags = [
+            {'class':['post-categories']},
             ]
 
-        feeds = []
-
-        for page in pages:
-            section, url, divider, attrList = page
-            newArticles = self.page_parse(url, divider, attrList)
-            feeds.append((section,newArticles))
-
-        # after the pages of the site have been processed, parse several RSS feeds for additional sections
-        newfeeds = Feed()
-        newfeeds = self.parse_rss()
-
-
-        # The utility code in parse_rss returns a Feed object.  Convert each feed/article combination into a form suitable
-        # for this module (parse_index).
-
-        for feed in newfeeds:
-            newArticles = []
-            for article in feed.articles:
-                newArt = {
-                            'title' : article.title,
-                            'url'   : article.url,
-                            'date'  : article.date,
-                            'description' : article.text_summary
-                        }
-                newArticles.append(newArt)
-
-
-            # New and Blogs should be the first two feeds.
-            if feed.title == 'New in RD':
-                feeds.insert(0,(feed.title,newArticles))
-            elif feed.title == 'Blogs':
-                feeds.insert(1,(feed.title,newArticles))
-            else:
-                feeds.append((feed.title,newArticles))
-
-
-        return feeds
-
-#-------------------------------------------------------------------------------------------------
-
-    def page_parse(self, mainurl, divider, attrList):
-
-        articles = []
-        mainsoup = self.index_to_soup(mainurl)
-        for item in mainsoup.findAll(attrs=attrList):
-            newArticle = {
-                        'title' : item('img')[0]['alt'],
-                        'url'   : 'http://www.rd.com'+item('a')[0]['href'],
-                        'date'  : '',
-                        'description' : ''
-                    }
-            articles.append(newArticle)
-
-
-
-        return articles
-
-
-
-#-------------------------------------------------------------------------------------------------
-
-    def parse_rss (self):
-
-        # Do the "official" parse_feeds first
-        feeds = BasicNewsRecipe.parse_feeds(self)
-
-
-        # Loop thru the articles in all feeds to find articles with "recipe" in it
-        recipeArticles = []
-        for curfeed in feeds:
-            delList = []
-            for a,curarticle in enumerate(curfeed.articles):
-                if curarticle.title.upper().find('RECIPE') >= 0:
-                    recipeArticles.append(curarticle)
-                    delList.append(curarticle)
-            if len(delList)>0:
-                for d in delList:
-                    index = curfeed.articles.index(d)
-                    curfeed.articles[index:index+1] = []
-
-        # If there are any recipes found, create a new Feed object and append.
-        if len(recipeArticles) > 0:
-            pfeed = Feed()
-            pfeed.title = 'Recipes'
-            pfeed.descrition = 'Recipe Feed (Virtual)'
-            pfeed.image_url  = None
-            pfeed.oldest_article = 30
-            pfeed.id_counter = len(recipeArticles)
-            # Create a new Feed, add the recipe articles, and then append
-            # to "official" list of feeds
-            pfeed.articles = recipeArticles[:]
-            feeds.append(pfeed)
-
-        return feeds
 

recipes/replicavedetelor.recipe

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, '
+'''
+replicavedetelor.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ReplicaVedetelor(BasicNewsRecipe):
+    title                 = u'Replica Vedetelor'
+    __author__            = u'Silviu Cotoara'
+    description           = u'Ofer\u0103 vedetelor dreptul la replic\u0103'
+    publisher             = 'Replica Vedetelor'
+    oldest_article        = 5
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare,Reviste,Vedete'
+    encoding              = 'utf-8'
+    cover_url       	  = 'http://www.webart-software.eu/_pics/lucrari_referinta/medium/84/1-Replica-Vedetelor.jpg'
+
+    conversion_options = {
+                             'comments'   : description
+                            ,'tags'       : category
+                            ,'language'   : language
+							,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+            dict(name='div', attrs={'id':'zona-continut'})
+                     ]
+
+    remove_tags = [
+             dict(name='ul', attrs={'id':['lista-imagini']})
+           , dict(name='form', attrs={'id':['f-trimite-unui-prieten']})
+
+                  ]
+
+    remove_tags_after = [
+              dict(name='form', attrs={'id':['f-trimite-unui-prieten']})
+            ]
+
+    feeds          = [
+            (u'Feeds', u'http://www.replicavedetelor.ro/feed')
+                 ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)
+

recipes/spektrum.recipe

@@ -0,0 +1,28 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+    title          = u'Spektrum (der Wissenschaft)'
+    __author__  = 'schuster'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    language               = 'de'
+    cover_url = 'http://upload.wikimedia.org/wikipedia/de/3/3b/Spektrum_der_Wissenschaft_Logo.svg'
+
+    remove_tags = [dict(attrs={'class':['hauptnaviPkt gainlayout', 'hauptnaviButton', 'suchButton', 'suchbegriffKasten', 'loginButton', 'subnavigation', 'artikelInfoLeiste gainlayout', 'artikelTools', 'nurLetzteSeite', 'link', 'boxUnterArtikel', 'leserbriefeBlock', 'boxTitel', 'boxInhalt', 'sehrklein', 'boxabstand', 'werbeboxinhalt', 'rbabstand', 'bildlinks', 'rechtebox', 'denkmalbox', 'denkmalfrage']}),
+	dict(id=['pflip', 'verlagsleiste', 'bereich', 'bannerVertikal', 'headerLogoLink', 'kopf', 'topNavi', 'headerSchnellsuche', 'headerSchnellsucheWarten', 'navigation', 'navigationL', 'navigationR', 'inhalt', 'rechtespalte', 'sdwboxenshop', 'shopboxen', 'fuss']),
+	dict(name=['naservice'])]
+
+    def print_version(self,url):
+        newurl = url.replace('artikel/', 'sixcms/detail.php?id=')
+        return newurl + '&_druckversion=1'
+
+
+
+    feeds          = [(u'Spektrum der Wissenschaft', u'http://www.spektrum.de/artikel/982623'),
+                          (u'SpektrumDirekt', u'http://www.spektrumdirekt.de/artikel/996406'),
+                          (u'Sterne und Weltraum', u'http://www.astronomie-heute.de/artikel/865248'),
+                          (u'Gehirn & Geist', u'http://www.gehirn-und-geist.de/artikel/982626'),
+                          (u'epoc', u'http://www.epoc.de/artikel/982625')
+
+]
+
+    filter_regexps = [r'ads\.doubleclick\.net']

recipes/strategy-business.recipe

@@ -33,7 +33,7 @@ class StrategyBusinessRecipe(BasicNewsRecipe):
                     elif c.name.endswith('_password'):
                         br[c.name] = self.password
                 raw = br.submit().read()
-                if '>Logout' not in raw:
+                if 'You have been logged in' not in raw:
                     raise ValueError('Failed to login, check your username and password')
         return br
 

recipes/technology_review_de.recipe

@@ -0,0 +1,24 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Technology Review'
+    __author__  = 'schuster'
+    remove_tags_before = dict(id='keywords')
+    remove_tags_after  = dict(id='kommentar')
+    remove_tags = [dict(attrs={'class':['navi_oben_pvg', 'navi_oben_tarifr', 'navi_oben_itm', 'navi_oben_eve', 'navi_oben_whi', 'navi_oben_abo', 'navi_oben_shop', 'navi_top_logo', 'navi_top_abschnitt', 'first']}),
+               dict(id=['footer', 'toolsRight', 'articleInline', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']),
+               dict(name=['script', 'noscript', 'style'])]
+    oldest_article = 4
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+
+    def print_version(self, url):
+        return url  + '?view=print'
+
+
+    feeds          = [
+    (u'Technik News', u'http://www.heise.de/tr/news-atom.xml') ]
+

recipes/united_daily.recipe

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+__license__   = 'GPL v3'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class UnitedDaily(BasicNewsRecipe):
+    title = u'聯合新聞網'
+    oldest_article = 1
+    max_articles_per_feed = 100
+
+    feeds = [(u'焦點', u'http://udn.com/udnrss/focus.xml'),
+             (u'政治', u'http://udn.com/udnrss/politics.xml'),
+             (u'社會', u'http://udn.com/udnrss/social.xml'),
+             (u'生活', u'http://udn.com/udnrss/life.xml'),
+             (u'綜合', u'http://udn.com/udnrss/education.xml'),
+             (u'意見評論', u'http://udn.com/udnrss/opinion.xml'),
+             (u'校園博覽會', u'http://mag.udn.com/udnrss/campus_rss.xml'),
+             (u'大台北', u'http://udn.com/udnrss/local_taipei.xml'),
+             (u'桃竹苗', u'http://udn.com/udnrss/local_tyhcml.xml'),
+             (u'中彰投', u'http://udn.com/udnrss/local_tcchnt.xml'),
+             (u'雲嘉南', u'http://udn.com/udnrss/local_ylcytn.xml'),
+             (u'高屏離島', u'http://udn.com/udnrss/local_ksptisland.xml'),
+             (u'基宜花東', u'http://udn.com/udnrss/local_klilhltt.xml'),
+             (u'台灣百寶鄉', u'http://udn.com/udnrss/local_oddlyenough.xml'),
+             (u'台灣人物', u'http://mag.udn.com/udnrss/people_rss.xml'),
+             (u'兩岸要聞', u'http://udn.com/udnrss/mainland.xml'),
+             (u'國際焦點', u'http://udn.com/udnrss/international.xml'),
+             (u'台商經貿', u'http://udn.com/udnrss/financechina.xml'),
+             (u'國際財經', u'http://udn.com/udnrss/financeworld.xml'),
+             (u'全球觀察', u'http://mag.udn.com/udnrss/world_rss.xml'),
+             (u'財經焦點', u'http://udn.com/udnrss/financesfocus.xml'),
+             (u'股市要聞', u'http://udn.com/udnrss/stock.xml'),
+             (u'股市快訊', u'http://udn.com/udnrss/stklatest.xml'),
+             (u'稅務法務', u'http://udn.com/udnrss/tax.xml'),
+             (u'房市情報', u'http://udn.com/udnrss/houses.xml'),
+             (u'個人理財', u'http://mag.udn.com/udnrss/wealth_rss.xml'),
+             (u'研究報告', u'http://mag.udn.com/udnrss/report_rss.xml'),
+             (u'基金', u'http://mag.udn.com/udnrss/fund_rss.xml'),
+             (u'理財會客室', u'http://mag.udn.com/udnrss/m_forum_rss.xml'),
+             (u'棒球', u'http://udn.com/udnrss/baseball.xml'),
+             (u'籃球', u'http://udn.com/udnrss/basketball.xml'),
+             (u'體壇動態', u'http://udn.com/udnrss/sportsfocus.xml'),
+             (u'熱門星聞', u'http://udn.com/udnrss/starsfocus.xml'),
+             (u'廣電港陸', u'http://udn.com/udnrss/tv.xml'),
+             (u'海外星球', u'http://udn.com/udnrss/starswestern.xml'),
+             (u'日韓星情', u'http://udn.com/udnrss/starsjk.xml'),
+             (u'電影世界', u'http://udn.com/udnrss/movie.xml'),
+             (u'流行音樂', u'http://udn.com/udnrss/music.xml'),
+             (u'觀點專題', u'http://udn.com/udnrss/starssubject.xml'),
+             (u'消費流行', u'http://mag.udn.com/udnrss/happylife_rss.xml'),
+             (u'食樂指南', u'http://udn.com/udnrss/food.xml'),
+             (u'數位資訊', u'http://mag.udn.com/udnrss/digital_rss.xml'),
+             (u'折扣好康', u'http://udn.com/udnrss/shopping.xml'),
+             (u'發燒車訊', u'http://mag.udn.com/udnrss/car_rss.xml'),
+             (u'醫藥新聞', u'http://udn.com/udnrss/health.xml'),
+             (u'家婦繽紛', u'http://udn.com/udnrss/benfen.xml'),
+             (u'談星論命', u'http://udn.com/udnrss/astrology.xml'),
+             (u'文化副刊', u'http://udn.com/udnrss/reading.xml'),
+             (u'旅遊休閒', u'http://travel.udn.com/udnrss/travel_rss.xml'),
+             (u'健康醫藥', u'http://mag.udn.com/udnrss/life_rss.xml'),
+             ]
+
+    extra_css = '''div[id='story_title'] {font-size:200%; font-weight:bold;} td[class='story_title'] {font-size:200%; font-weight:bold;} td[class='story_title'] td[class='story_title']>div {font-size:200%; font-weight:bold;}'''
+
+    __author__ = 'Eddie Lau'
+    __version__ = '1.1'
+    language = 'zh-TW'
+    publisher = 'United Daily News Group'
+    description = 'United Daily (Taiwan)'
+    category = 'News, Chinese, Taiwan'
+    remove_javascript = True
+    use_embedded_content = False
+    no_stylesheets = True
+    encoding = 'big5'
+    conversion_options = {'linearize_tables':True}
+    masthead_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif'
+    cover_url = 'http://udn.com/NEWS/2004/images/logo_udn.gif'
+    keep_only_tags = [dict(name='td', attrs={'class':['story_title']}),
+                      dict(name='div', attrs={'id':['story_title']}),
+                      dict(name='td', attrs={'class':['story_author']}),
+                      dict(name='div', attrs={'id':['story_author']}),
+                      dict(name='td', attrs={'class':['story']}),
+                      dict(name='div', attrs={'id':['story']}),
+                      ]
+    remove_tags = [dict(name='div', attrs={'id':['mvouter']})]
+

recipes/welt_der_physik.recipe

@@ -0,0 +1,20 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1303841067(BasicNewsRecipe):
+
+    title          = u'Welt der Physik'
+    __author__  = 'schuster'
+    remove_tags_befor = [dict(name='div', attrs={'class':'inhalt_bild_text_printonly'})]
+    remove_tags_after = [dict(name='span', attrs={'class':'clearinhalt_bild'})]
+    remove_tags = [dict(attrs={'class':['invisible', 'searchfld', 'searchbtn', 'topnavi', 'topsearch']}),
+	dict(id=['naservice', 'phservicemenu', '',]),
+	dict(name=['naservice'])]
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets         = True
+    use_embedded_content   = False
+    language               = 'de'
+    remove_javascript      = True
+
+
+    feeds          = [(u'Nachrichten und Neuigkeiten', u'http://www.weltderphysik.de/rss/alles.xml')]

resources/default_tweaks.py

@@ -41,14 +41,19 @@ authors_completer_append_separator = False
 #: Author sort name algorithm
 # The algorithm used to copy author to author_sort
 # Possible values are:
-#  invert: use "fn ln" -> "ln, fn" (the default algorithm)
+#  invert: use "fn ln" -> "ln, fn"
 #  copy  : copy author to author_sort without modification
 #  comma : use 'copy' if there is a ',' in the name, otherwise use 'invert'
 #  nocomma : "fn ln" -> "ln fn" (without the comma)
 # When this tweak is changed, the author_sort values stored with each author
 # must be recomputed by right-clicking on an author in the left-hand tags pane,
 # selecting 'manage authors', and pressing 'Recalculate all author sort values'.
+# The author name suffixes are words that are ignored when they occur at the
+# end of an author name. The case of the suffix is ignored and trailing
+# periods are automatically handled.
 author_sort_copy_method = 'comma'
+author_name_suffixes = ('Jr', 'Sr', 'Inc', 'Ph.D', 'Phd',
+                        'MD', 'M.D', 'I', 'II', 'III', 'IV')
 
 #: Use author sort in Tag Browser
 # Set which author field to display in the tags pane (the list of authors,
@@ -345,3 +350,11 @@ send_news_to_device_location = "main"
 # work on all operating systems)
 server_listen_on = '0.0.0.0'
 
+#: Unified toolbar on OS X
+# If you enable this option and restart calibre, the toolbar will be 'unified'
+# with the titlebar as is normal for OS X applications. However, doing this has
+# various bugs, for instance the minimum width of the toolbar becomes twice
+# what it should be and it causes other random bugs on some systems, so turn it
+# on at your own risk!
+unified_title_toolbar_on_osx = False
+

setup/commands.py

@@ -11,7 +11,7 @@ __all__ = [
         'build', 'build_pdf2xml', 'server',
         'gui',
         'develop', 'install',
-        'resources',
+        'kakasi', 'resources',
         'check',
         'sdist',
         'manual', 'tag_release',
@@ -49,8 +49,9 @@ gui = GUI()
 from setup.check import Check
 check = Check()
 
-from setup.resources import Resources
+from setup.resources import Resources, Kakasi
 resources = Resources()
+kakasi = Kakasi()
 
 from setup.publish import Manual, TagRelease, Stage1, Stage2, \
         Stage3, Stage4, Publish

setup/installer/linux/util.c

@@ -30,11 +30,12 @@ int report_libc_error(const char *msg) {
 }
 
 int pyobject_to_int(PyObject *res) {
-    int ret; PyObject *tmp;
-    tmp = PyNumber_Int(res);
-    if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0;
-    else ret = (int)PyInt_AS_LONG(tmp);
-
+    int ret = 0; PyObject *tmp;
+    if (res != NULL) {
+        tmp = PyNumber_Int(res);
+        if (tmp == NULL) ret = (PyObject_IsTrue(res)) ? 1 : 0;
+        else ret = (int)PyInt_AS_LONG(tmp);
+    }
     return ret;
 }
 

setup/installer/windows/freeze.py

@@ -14,7 +14,7 @@ from setup.build_environment import msvc, MT, RC
 from setup.installer.windows.wix import WixMixIn
 
 OPENSSL_DIR = r'Q:\openssl'
-QT_DIR = 'Q:\\Qt\\4.7.2'
+QT_DIR = 'Q:\\Qt\\4.7.3'
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
 LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'

setup/installer/windows/wix-template.xml

@@ -11,9 +11,6 @@
 						SummaryCodepage='1252' />
 
                 <Media Id="1" Cabinet="{app}.cab" CompressionLevel="{compression}" EmbedCab="yes" />
-                <!-- The following line is needed because of the patch to QtCore4.dll. You can remove this line
-                     after you update Qt beyond 4.7.2. 'emus' means re-install even if version is the same not just if it is older. -->
-                <Property Id='REINSTALLMODE' Value='emus'/>
 
                 <Upgrade Id="{upgrade_code}">
                     <UpgradeVersion Maximum="{version}"

setup/resources.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, cPickle, re, anydbm, shutil, marshal, zipfile, glob
+import os, cPickle, re, shutil, marshal, zipfile, glob
 from zlib import compress
 
 from setup import Command, basenames, __appname__
@@ -23,13 +23,114 @@ def get_opts_from_parser(parser):
         for o in g.option_list:
             for x in do_opt(o): yield x
 
-class Resources(Command):
+class Kakasi(Command):
 
-    description = 'Compile various needed calibre resources'
+    description = 'Compile resources for unihandecode'
 
     KAKASI_PATH = os.path.join(Command.SRC,  __appname__,
             'ebooks', 'unihandecode', 'pykakasi')
 
+    def run(self, opts):
+        self.records = {}
+        src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','kanwadict2.pickle')
+        base = os.path.dirname(dest)
+        if not os.path.exists(base):
+            os.makedirs(base)
+
+        if self.newer(dest, src):
+            self.info('\tGenerating Kanwadict')
+
+            for line in open(src, "r"):
+                self.parsekdict(line)
+            self.kanwaout(dest)
+
+        src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','itaijidict2.pickle')
+
+        if self.newer(dest, src):
+            self.info('\tGenerating Itaijidict')
+            self.mkitaiji(src, dest)
+
+        src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
+        dest = self.j(self.RESOURCES, 'localization',
+                'pykakasi','kanadict2.pickle')
+
+        if self.newer(dest, src):
+            self.info('\tGenerating kanadict')
+            self.mkkanadict(src, dest)
+
+        return
+
+
+    def mkitaiji(self, src, dst):
+        dic = {}
+        for line in open(src, "r"):
+            line = line.decode("utf-8").strip()
+            if line.startswith(';;'): # skip comment
+                continue
+            if re.match(r"^$",line):
+                continue
+            pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
+            dic[pair[0]] = pair[1]
+        cPickle.dump(dic, open(dst, 'wb'), protocol=-1) #pickle
+
+    def mkkanadict(self, src, dst):
+        dic = {}
+        for line in open(src, "r"):
+            line = line.decode("utf-8").strip()
+            if line.startswith(';;'): # skip comment
+                continue
+            if re.match(r"^$",line):
+                continue
+            (alpha, kana) = line.split(' ')
+            dic[kana] = alpha
+        cPickle.dump(dic, open(dst, 'wb'), protocol=-1) #pickle
+
+    def parsekdict(self, line):
+        line = line.decode("utf-8").strip()
+        if line.startswith(';;'): # skip comment
+            return
+        (yomi, kanji) = line.split(' ')
+        if ord(yomi[-1:]) <= ord('z'):
+            tail = yomi[-1:]
+            yomi = yomi[:-1]
+        else:
+            tail = ''
+        self.updaterec(kanji, yomi, tail)
+
+    def updaterec(self, kanji, yomi, tail):
+        key = "%04x"%ord(kanji[0])
+        if key in self.records:
+            if kanji in self.records[key]:
+                rec = self.records[key][kanji]
+                rec.append((yomi,tail))
+                self.records[key].update( {kanji: rec} )
+            else:
+                self.records[key][kanji]=[(yomi, tail)]
+        else:
+            self.records[key] = {}
+            self.records[key][kanji]=[(yomi, tail)]
+
+    def kanwaout(self, out):
+        with open(out, 'wb') as f:
+            dic = {}
+            for k, v in self.records.iteritems():
+                dic[k] = compress(marshal.dumps(v))
+            cPickle.dump(dic, f, -1)
+
+    def clean(self):
+        kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')
+        if os.path.exists(kakasi):
+            shutil.rmtree(kakasi)
+
+class Resources(Command):
+
+    description = 'Compile various needed calibre resources'
+    sub_commands = ['kakasi']
+
     def run(self, opts):
         scripts = {}
         for x in ('console', 'gui'):
@@ -117,108 +218,13 @@ class Resources(Command):
         import json
         json.dump(function_dict, open(dest, 'wb'), indent=4)
 
-        self.run_kakasi(opts)
-
-    def run_kakasi(self, opts):
-        self.records = {}
-        src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanwadict2.db')
-        base = os.path.dirname(dest)
-        if not os.path.exists(base):
-            os.makedirs(base)
-
-        if self.newer(dest, src):
-            self.info('\tGenerating Kanwadict')
-
-            for line in open(src, "r"):
-                self.parsekdict(line)
-            self.kanwaout(dest)
-
-        src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','itaijidict2.pickle')
-
-        if self.newer(dest, src):
-            self.info('\tGenerating Itaijidict')
-            self.mkitaiji(src, dest)
-
-        src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
-        dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanadict2.pickle')
-
-        if self.newer(dest, src):
-            self.info('\tGenerating kanadict')
-            self.mkkanadict(src, dest)
-
-        return
-
-
-    def mkitaiji(self, src, dst):
-        dic = {}
-        for line in open(src, "r"):
-            line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
-                continue
-            if re.match(r"^$",line):
-                continue
-            pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
-            dic[pair[0]] = pair[1]
-        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
-
-    def mkkanadict(self, src, dst):
-        dic = {}
-        for line in open(src, "r"):
-            line = line.decode("utf-8").strip()
-            if line.startswith(';;'): # skip comment
-                continue
-            if re.match(r"^$",line):
-                continue
-            (alpha, kana) = line.split(' ')
-            dic[kana] = alpha
-        cPickle.dump(dic, open(dst, 'w'), protocol=-1) #pickle
-
-    def parsekdict(self, line):
-        line = line.decode("utf-8").strip()
-        if line.startswith(';;'): # skip comment
-            return
-        (yomi, kanji) = line.split(' ')
-        if ord(yomi[-1:]) <= ord('z'):
-            tail = yomi[-1:]
-            yomi = yomi[:-1]
-        else:
-            tail = ''
-        self.updaterec(kanji, yomi, tail)
-
-    def updaterec(self, kanji, yomi, tail):
-            key = "%04x"%ord(kanji[0])
-            if key in self.records:
-                if kanji in self.records[key]:
-                    rec = self.records[key][kanji]
-                    rec.append((yomi,tail))
-                    self.records[key].update( {kanji: rec} )
-                else:
-                    self.records[key][kanji]=[(yomi, tail)]
-            else:
-                self.records[key] = {}
-                self.records[key][kanji]=[(yomi, tail)]
-
-    def kanwaout(self, out):
-        dic = anydbm.open(out, 'c')
-        for (k, v) in self.records.iteritems():
-            dic[k] = compress(marshal.dumps(v))
-        dic.close()
-
-
     def clean(self):
         for x in ('scripts', 'recipes', 'ebook-convert-complete'):
             x = self.j(self.RESOURCES, x+'.pickle')
             if os.path.exists(x):
                 os.remove(x)
-        kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')
-        if os.path.exists(kakasi):
-            shutil.rmtree(kakasi)
-
+        from setup.commands import kakasi
+        kakasi.clean()
 
 
 

src/calibre/__init__.py

@@ -630,6 +630,24 @@ def human_readable(size):
         size = size[:-2]
     return size + " " + suffix
 
+def remove_bracketed_text(src,
+        brackets={u'(':u')', u'[':u']', u'{':u'}'}):
+    from collections import Counter
+    counts = Counter()
+    buf = []
+    src = force_unicode(src)
+    rmap = dict([(v, k) for k, v in brackets.iteritems()])
+    for char in src:
+        if char in brackets:
+            counts[char] += 1
+        elif char in rmap:
+            idx = rmap[char]
+            if counts[idx] > 0:
+                counts[idx] -= 1
+        elif sum(counts.itervalues()) < 1:
+            buf.append(char)
+    return u''.join(buf)
+
 if isosx:
     import glob, shutil
     fdir = os.path.expanduser('~/.fonts')

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 0)
+numeric_version = (0, 8, 1)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -1,4 +1,5 @@
-import os.path
+# -*- coding: utf-8 -*-
+
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
@@ -628,8 +629,9 @@ from calibre.ebooks.metadata.sources.amazon import Amazon
 from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
 from calibre.ebooks.metadata.sources.isbndb import ISBNDB
 from calibre.ebooks.metadata.sources.overdrive import OverDrive
+from calibre.ebooks.metadata.sources.douban import Douban
 
-plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive]
+plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban]
 
 # }}}
 
@@ -1093,14 +1095,25 @@ plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
 # Store plugins {{{
 class StoreAmazonKindleStore(StoreBase):
     name = 'Amazon Kindle'
-    description = _('Kindle books from Amazon')
+    description = _('Kindle books from Amazon.')
     actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore'
 
+class StoreAmazonDEKindleStore(StoreBase):
+    name = 'Amazon DE Kindle'
+    description = _('Kindle books from Amazon.de.')
+    actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore'
+
 class StoreAmazonUKKindleStore(StoreBase):
     name = 'Amazon UK Kindle'
-    description = _('Kindle books from Amazon.uk')
+    description = _('Kindle books from Amazon.uk.')
     actual_plugin = 'calibre.gui2.store.amazon_uk_plugin:AmazonUKKindleStore'
 
+class StoreArchiveOrgStore(StoreBase):
+    name = 'Archive.org'
+    description = _('Free Books : Download & Streaming : Ebook  and Texts Archive : Internet Archive.')
+    actual_plugin = 'calibre.gui2.store.archive_org_plugin:ArchiveOrgStore'
+
+
 class StoreBaenWebScriptionStore(StoreBase):
     name = 'Baen WebScription'
     description = _('Ebooks for readers.')
@@ -1111,6 +1124,11 @@ class StoreBNStore(StoreBase):
     description = _('Books, Textbooks, eBooks, Toys, Games and More.')
     actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore'
 
+class StoreBeamEBooksDEStore(StoreBase):
+    name = 'Beam EBooks DE'
+    description = _('Der eBook Shop.')
+    actual_plugin = 'calibre.gui2.store.beam_ebooks_de_plugin:BeamEBooksDEStore'
+
 class StoreBeWriteStore(StoreBase):
     name = 'BeWrite Books'
     description = _('Publishers of fine books.')
@@ -1126,9 +1144,14 @@ class StoreEbookscomStore(StoreBase):
     description = _('The digital bookstore.')
     actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore'
 
+class StoreEPubBuyDEStore(StoreBase):
+    name = 'EPUBBuy DE'
+    description = _('EPUBReaders eBook Shop.')
+    actual_plugin = 'calibre.gui2.store.epubbuy_de_plugin:EPubBuyDEStore'
+
 class StoreEHarlequinStore(StoreBase):
     name = 'eHarlequin'
-    description = _('entertain, enrich, inspire.')
+    description = _('Entertain, enrich, inspire.')
     actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore'
 
 class StoreFeedbooksStore(StoreBase):
@@ -1136,6 +1159,22 @@ class StoreFeedbooksStore(StoreBase):
     description = _('Read anywhere.')
     actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore'
 
+class StoreFoylesUKStore(StoreBase):
+    name = 'Foyles UK'
+    description = _('Foyles of London, online.')
+    actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore'
+
+class StoreGandalfStore(StoreBase):
+    name = 'Gandalf'
+    author = 'Tomasz Długosz'
+    description = _('Zaczarowany świat książek')
+    actual_plugin = 'calibre.gui2.store.gandalf_plugin:GandalfStore'
+
+class StoreGoogleBooksStore(StoreBase):
+    name = 'Google Books'
+    description = _('Google Books')
+    actual_plugin = 'calibre.gui2.store.google_books_plugin:GoogleBooksStore'
+
 class StoreGutenbergStore(StoreBase):
     name = 'Project Gutenberg'
     description = _('The first producer of free ebooks.')
@@ -1153,14 +1192,30 @@ class StoreManyBooksStore(StoreBase):
 
 class StoreMobileReadStore(StoreBase):
     name = 'MobileRead'
-    description = _('Ebooks handcrafted with the utmost care')
+    description = _('Ebooks handcrafted with the utmost care.')
     actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore'
 
+class StoreNextoStore(StoreBase):
+    name = 'Nexto'
+    author = 'Tomasz Długosz'
+    description = _('Audiobooki mp3, ebooki, prasa - księgarnia internetowa.')
+    actual_plugin = 'calibre.gui2.store.nexto_plugin:NextoStore'
+
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'
     description = _('One web page for every book.')
     actual_plugin = 'calibre.gui2.store.open_library_plugin:OpenLibraryStore'
 
+class StoreOReillyStore(StoreBase):
+    name = 'OReilly'
+    description = _('DRM-Free tech ebooks.')
+    actual_plugin = 'calibre.gui2.store.oreilly_plugin:OReillyStore'
+
+class StorePragmaticBookshelfStore(StoreBase):
+    name = 'Pragmatic Bookshelf'
+    description = _('The Pragmatic Bookshelf')
+    actual_plugin = 'calibre.gui2.store.pragmatic_bookshelf_plugin:PragmaticBookshelfStore'
+
 class StoreSmashwordsStore(StoreBase):
     name = 'Smashwords'
     description = _('Your ebook. Your way.')
@@ -1168,36 +1223,48 @@ class StoreSmashwordsStore(StoreBase):
 
 class StoreWaterstonesUKStore(StoreBase):
     name = 'Waterstones UK'
-    description = _('Feel every word')
+    description = _('Feel every word.')
     actual_plugin = 'calibre.gui2.store.waterstones_uk_plugin:WaterstonesUKStore'
 
-class StoreFoylesUKStore(StoreBase):
-    name = 'Foyles UK'
-    description = _('Foyles of London, online')
-    actual_plugin = 'calibre.gui2.store.foyles_uk_plugin:FoylesUKStore'
+class StoreWeightlessBooksStore(StoreBase):
+    name = 'Weightless Books'
+    description = '(e)Books That Don\'t Weigh You Down.'
+    actual_plugin = 'calibre.gui2.store.weightless_books_plugin:WeightlessBooksStore'
 
-class StoreAmazonDEKindleStore(StoreBase):
-    name = 'Amazon DE Kindle'
-    description = _('Kindle eBooks')
-    actual_plugin = 'calibre.gui2.store.amazon_de_plugin:AmazonDEKindleStore'
+class StoreWizardsTowerBooksStore(StoreBase):
+    name = 'Wizards Tower Books'
+    description = 'Wizard\'s Tower Press.'
+    actual_plugin = 'calibre.gui2.store.wizards_tower_books_plugin:WizardsTowerBooksStore'
 
-class StoreBeamEBooksDEStore(StoreBase):
-    name = 'Beam EBooks DE'
-    description = _('der eBook Shop')
-    actual_plugin = 'calibre.gui2.store.beam_ebooks_de_plugin:BeamEBooksDEStore'
-
-class StoreEPubBuyDEStore(StoreBase):
-    name = 'EPUBBuy DE'
-    description = _('EPUBReaders eBook Shop')
-    actual_plugin = 'calibre.gui2.store.epubbuy_de_plugin:EPubBuyDEStore'
-
-plugins += [StoreAmazonKindleStore, StoreAmazonDEKindleStore,
+plugins += [
+    StoreArchiveOrgStore,
+    StoreAmazonKindleStore,
+    StoreAmazonDEKindleStore,
     StoreAmazonUKKindleStore,
-    StoreBaenWebScriptionStore, StoreBNStore, StoreBeamEBooksDEStore,
-    StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore,
-    StoreEPubBuyDEStore, StoreEHarlequinStore, StoreFeedbooksStore,
-    StoreFoylesUKStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore,
-    StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore,
-    StoreWaterstonesUKStore]
+    StoreBaenWebScriptionStore,
+    StoreBNStore,
+    StoreBeamEBooksDEStore,
+    StoreBeWriteStore,
+    StoreDieselEbooksStore,
+    StoreEbookscomStore,
+    StoreEPubBuyDEStore,
+    StoreEHarlequinStore,
+    StoreFeedbooksStore,
+    StoreFoylesUKStore,
+    StoreGandalfStore, 
+    StoreGoogleBooksStore,
+    StoreGutenbergStore,
+    StoreKoboStore,
+    StoreManyBooksStore,
+    StoreMobileReadStore,
+    StoreNextoStore,
+    StoreOpenLibraryStore,
+    StoreOReillyStore,
+    StorePragmaticBookshelfStore,
+    StoreSmashwordsStore,
+    StoreWaterstonesUKStore,
+    StoreWeightlessBooksStore,
+    StoreWizardsTowerBooksStore
+]
 
 # }}}

src/calibre/customize/profiles.py

@@ -253,7 +253,7 @@ class OutputProfile(Plugin):
     periodical_date_in_title = True
 
     #: Characters used in jackets and catalogs
-	missing_char = u'x'
+    missing_char = u'x'
     ratings_char = u'*'
     empty_ratings_char = u' '
     read_char = u'+'
@@ -293,38 +293,38 @@ class iPadOutput(OutputProfile):
         }
     ]
 
-	missing_char = u'\u2715\u200a'		# stylized 'x' plus hair space
-    ratings_char = u'\u2605'			# filled star
-	empty_ratings_char = u'\u2606'		# hollow star
-    read_char = u'\u2713'				# check mark
+    missing_char = u'\u2715\u200a'      # stylized 'x' plus hair space
+    ratings_char = u'\u2605'            # filled star
+    empty_ratings_char = u'\u2606'      # hollow star
+    read_char = u'\u2713'               # check mark
 
     touchscreen = True
     # touchscreen_news_css {{{
     touchscreen_news_css = u'''
-			/* hr used in articles */
-			.article_articles_list {
+            /* hr used in articles */
+            .article_articles_list {
                 width:18%;
-				}
+                }
             .article_link {
-            	color: #593f29;
+                color: #593f29;
                 font-style: italic;
                 }
             .article_next {
-				-webkit-border-top-right-radius:4px;
-				-webkit-border-bottom-right-radius:4px;
+                -webkit-border-top-right-radius:4px;
+                -webkit-border-bottom-right-radius:4px;
                 font-style: italic;
                 width:32%;
                 }
 
             .article_prev {
-				-webkit-border-top-left-radius:4px;
-				-webkit-border-bottom-left-radius:4px;
+                -webkit-border-top-left-radius:4px;
+                -webkit-border-bottom-left-radius:4px;
                 font-style: italic;
                 width:32%;
                 }
-			.article_sections_list {
+            .article_sections_list {
                 width:18%;
-				}
+                }
             .articles_link {
                 font-weight: bold;
                 }
@@ -334,8 +334,8 @@ class iPadOutput(OutputProfile):
 
 
             .caption_divider {
-            	border:#ccc 1px solid;
-				}
+                border:#ccc 1px solid;
+                }
 
             .touchscreen_navbar {
                 background:#c3bab2;
@@ -357,50 +357,50 @@ class iPadOutput(OutputProfile):
                 text-align:center;
                 }
 
-			.touchscreen_navbar td a:link {
-				color: #593f29;
-				text-decoration: none;
-				}
+            .touchscreen_navbar td a:link {
+                color: #593f29;
+                text-decoration: none;
+                }
 
-			/* Index formatting */
-			.publish_date {
-				text-align:center;
-				}
-			.divider {
-				border-bottom:1em solid white;
-				border-top:1px solid gray;
-				}
+            /* Index formatting */
+            .publish_date {
+                text-align:center;
+                }
+            .divider {
+                border-bottom:1em solid white;
+                border-top:1px solid gray;
+                }
 
-			hr.caption_divider {
-				border-color:black;
-				border-style:solid;
-				border-width:1px;
-				}
+            hr.caption_divider {
+                border-color:black;
+                border-style:solid;
+                border-width:1px;
+                }
 
             /* Feed summary formatting */
             .article_summary {
-            	display:inline-block;
-            	}
+                display:inline-block;
+                }
             .feed {
                 font-family:sans-serif;
                 font-weight:bold;
                 font-size:larger;
-				}
+                }
 
             .feed_link {
                 font-style: italic;
                 }
 
             .feed_next {
-				-webkit-border-top-right-radius:4px;
-				-webkit-border-bottom-right-radius:4px;
+                -webkit-border-top-right-radius:4px;
+                -webkit-border-bottom-right-radius:4px;
                 font-style: italic;
                 width:40%;
                 }
 
             .feed_prev {
-				-webkit-border-top-left-radius:4px;
-				-webkit-border-bottom-left-radius:4px;
+                -webkit-border-top-left-radius:4px;
+                -webkit-border-bottom-left-radius:4px;
                 font-style: italic;
                 width:40%;
                 }
@@ -410,24 +410,24 @@ class iPadOutput(OutputProfile):
                 font-size: 160%;
                 }
 
-			.feed_up {
+            .feed_up {
                 font-weight: bold;
                 width:20%;
-				}
+                }
 
             .summary_headline {
                 font-weight:bold;
                 text-align:left;
-				}
+                }
 
             .summary_byline {
                 text-align:left;
                 font-family:monospace;
-				}
+                }
 
             .summary_text {
                 text-align:left;
-				}
+                }
 
         '''
         # }}}
@@ -617,8 +617,8 @@ class KindleOutput(OutputProfile):
     supports_mobi_indexing = True
     periodical_date_in_title = False
 
-	missing_char = u'x\u2009'
-	empty_ratings_char = u'\u2606'
+    missing_char = u'x\u2009'
+    empty_ratings_char = u'\u2606'
     ratings_char = u'\u2605'
     read_char = u'\u2713'
 
@@ -642,8 +642,8 @@ class KindleDXOutput(OutputProfile):
     #comic_screen_size         = (741, 1022)
     supports_mobi_indexing = True
     periodical_date_in_title = False
-	missing_char = u'x\u2009'
-	empty_ratings_char = u'\u2606'
+    missing_char = u'x\u2009'
+    empty_ratings_char = u'\u2606'
     ratings_char = u'\u2605'
     read_char = u'\u2713'
     mobi_ems_per_blockquote = 2.0

src/calibre/customize/ui.py

@@ -92,7 +92,7 @@ def restore_plugin_state_to_default(plugin_or_name):
     config['enabled_plugins'] = ep
 
 default_disabled_plugins = set([
-    'Overdrive',
+    'Overdrive', 'Douban Books',
 ])
 
 def is_disabled(plugin):

src/calibre/devices/android/driver.py

@@ -109,7 +109,7 @@ class ANDROID(USBMS):
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
-            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB']
+            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD']

src/calibre/devices/apple/driver.py

@@ -941,7 +941,7 @@ class ITUNES(DriverBase):
         # declared in use_plugboard_ext and a device name of ITUNES
         if DEBUG:
             self.log.info("ITUNES.set_plugboard()")
-            #self.log.info('  using plugboard %s' % plugboards)
+            #self.log.info('  plugboard: %s' % plugboards)
         self.plugboards = plugboards
         self.plugboard_func = pb_func
 
@@ -1052,7 +1052,6 @@ class ITUNES(DriverBase):
                     'title': metadata[i].title,
                      'uuid': metadata[i].uuid }
 
-
                 # Report progress
                 if self.report_progress is not None:
                     self.report_progress((i+1)/file_count, _('%d of %d') % (i+1, file_count))
@@ -2744,7 +2743,7 @@ class ITUNES(DriverBase):
         # Update metadata from plugboard
         # If self.plugboard is None (no transforms), original metadata is returned intact
         metadata_x = self._xform_metadata_via_plugboard(metadata, this_book.format)
-
+        self.log("metadata.title_sort: %s  metadata_x.title_sort: %s" % (metadata.title_sort, metadata_x.title_sort))
         if isosx:
             if lb_added:
                 lb_added.name.set(metadata_x.title)
@@ -2754,8 +2753,7 @@ class ITUNES(DriverBase):
                 lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.enabled.set(True)
                 lb_added.sort_artist.set(icu_title(metadata_x.author_sort))
-                lb_added.sort_name.set(metadata.title_sort)
-
+                lb_added.sort_name.set(metadata_x.title_sort)
 
             if db_added:
                 db_added.name.set(metadata_x.title)
@@ -2765,7 +2763,7 @@ class ITUNES(DriverBase):
                 db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.enabled.set(True)
                 db_added.sort_artist.set(icu_title(metadata_x.author_sort))
-                db_added.sort_name.set(metadata.title_sort)
+                db_added.sort_name.set(metadata_x.title_sort)
 
             if metadata_x.comments:
                 if lb_added:
@@ -2785,6 +2783,7 @@ class ITUNES(DriverBase):
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
+            # If title_sort applied in plugboard, that overrides using series/index as title_sort
             if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]:
                 if DEBUG:
                     self.log.info(" ITUNES._update_iTunes_metadata()")
@@ -2796,7 +2795,9 @@ class ITUNES(DriverBase):
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
-                    lb_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
+                        lb_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
                     lb_added.episode_ID.set(metadata_x.series)
                     lb_added.episode_number.set(metadata_x.series_index)
 
@@ -2810,7 +2811,9 @@ class ITUNES(DriverBase):
                                 break
 
                 if db_added:
-                    db_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
+                        db_added.sort_name.set("%s %s" % (self.title_sorter(metadata_x.series), series_index))
                     db_added.episode_ID.set(metadata_x.series)
                     db_added.episode_number.set(metadata_x.series_index)
 
@@ -2845,7 +2848,7 @@ class ITUNES(DriverBase):
                 lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 lb_added.Enabled = True
                 lb_added.SortArtist = icu_title(metadata_x.author_sort)
-                lb_added.SortName = metadata.title_sort
+                lb_added.SortName = metadata_x.title_sort
 
             if db_added:
                 db_added.Name = metadata_x.title
@@ -2855,7 +2858,7 @@ class ITUNES(DriverBase):
                 db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S')))
                 db_added.Enabled = True
                 db_added.SortArtist = icu_title(metadata_x.author_sort)
-                db_added.SortName = metadata.title_sort
+                db_added.SortName = metadata_x.title_sort
 
             if metadata_x.comments:
                 if lb_added:
@@ -2888,7 +2891,9 @@ class ITUNES(DriverBase):
                 fraction = index-integer
                 series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0'))
                 if lb_added:
-                    lb_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
+                        lb_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
                     lb_added.EpisodeID = metadata_x.series
 
                     try:
@@ -2914,7 +2919,9 @@ class ITUNES(DriverBase):
                                 break
 
                 if db_added:
-                    db_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
+                    # If no title_sort plugboard tweak, create sort_name from series/index
+                    if metadata.title_sort == metadata_x.title_sort:
+                        db_added.SortName = "%s %s" % (self.title_sorter(metadata_x.series), series_index)
                     db_added.EpisodeID = metadata_x.series
 
                     try:
@@ -2975,6 +2982,9 @@ class ITUNES(DriverBase):
                                            newmi.publisher if book.publisher != newmi.publisher else ''))
                 self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
                                            newmi.tags if book.tags != newmi.tags else ''))
+            else:
+                self.log("  matching plugboard not found")
+
         else:
             newmi = book
         return newmi

src/calibre/devices/kobo/driver.py

@@ -38,7 +38,7 @@ class KOBO(USBMS):
 
     VENDOR_ID   = [0x2237]
     PRODUCT_ID  = [0x4161]
-    BCD         = [0x0110]
+    BCD         = [0x0110, 0x0323]
 
     VENDOR_NAME = ['KOBO_INC', 'KOBO']
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['.KOBOEREADER', 'EREADER']

src/calibre/ebooks/epub/input.py

@@ -103,10 +103,11 @@ class EPUBInput(InputFormatPlugin):
         t.set('href', guide_cover)
         t.set('title', 'Title Page')
         from calibre.ebooks import render_html_svg_workaround
-        renderer = render_html_svg_workaround(guide_cover, log)
-        if renderer is not None:
-            open('calibre_raster_cover.jpg', 'wb').write(
-                renderer)
+        if os.path.exists(guide_cover):
+            renderer = render_html_svg_workaround(guide_cover, log)
+            if renderer is not None:
+                open('calibre_raster_cover.jpg', 'wb').write(
+                    renderer)
 
     def find_opf(self):
         def attr(n, attr):

src/calibre/ebooks/metadata/__init__.py

@@ -10,7 +10,7 @@ import os, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath, guess_type
+from calibre import relpath, guess_type, remove_bracketed_text
 
 from calibre.utils.config import tweaks
 
@@ -27,20 +27,37 @@ def authors_to_string(authors):
     else:
         return ''
 
-_bracket_pat = re.compile(r'[\[({].*?[})\]]')
-def author_to_author_sort(author):
+def author_to_author_sort(author, method=None):
     if not author:
-        return ''
-    method = tweaks['author_sort_copy_method']
-    if method == 'copy' or (method == 'comma' and ',' in author):
+        return u''
+    sauthor = remove_bracketed_text(author).strip()
+    tokens = sauthor.split()
+    if len(tokens) < 2:
         return author
-    author = _bracket_pat.sub('', author).strip()
-    tokens = author.split()
-    if tokens and tokens[-1] not in ('Inc.', 'Inc'):
-        tokens = tokens[-1:] + tokens[:-1]
-        if len(tokens) > 1 and method != 'nocomma':
-            tokens[0] += ','
-    return ' '.join(tokens)
+    if method is None:
+        method = tweaks['author_sort_copy_method']
+    if method == u'copy':
+        return author
+    suffixes = set([x.lower() for x in tweaks['author_name_suffixes']])
+    suffixes |= set([x+u'.' for x in suffixes])
+
+    last = tokens[-1].lower()
+    suffix = None
+    if last in suffixes:
+        suffix = tokens[-1]
+        tokens = tokens[:-1]
+
+    if method == u'comma' and u',' in u''.join(tokens):
+        return author
+
+    atokens = tokens[-1:] + tokens[:-1]
+    if suffix:
+        atokens.append(suffix)
+
+    if method != u'nocomma' and len(atokens) > 1:
+        atokens[0] += u','
+
+    return u' '.join(atokens)
 
 def authors_to_sort_string(authors):
     return ' & '.join(map(author_to_author_sort, authors))

src/calibre/ebooks/metadata/archive.py

@@ -83,6 +83,7 @@ class ArchiveExtract(FileTypePlugin):
         return of.name
 
 def get_comic_book_info(d, mi):
+    # See http://code.google.com/p/comicbookinfo/wiki/Example
     series = d.get('series', '')
     if series.strip():
         mi.series = series
@@ -111,6 +112,7 @@ def get_comic_book_info(d, mi):
 
 
 def get_cbz_metadata(stream):
+    # See http://code.google.com/p/comicbookinfo/wiki/Example
     from calibre.utils.zipfile import ZipFile
     from calibre.ebooks.metadata import MetaInformation
     import json

src/calibre/ebooks/metadata/book/base.py

@@ -112,10 +112,15 @@ class Metadata(object):
 
         Be careful with numeric fields since this will return True for zero as
         well as None.
+
+        Also returns True if the field does not exist.
         '''
-        null_val = NULL_VALUES.get(field, None)
-        val = getattr(self, field, None)
-        return not val or val == null_val
+        try:
+            null_val = NULL_VALUES.get(field, None)
+            val = getattr(self, field, None)
+            return not val or val == null_val
+        except:
+            return True
 
     def __getattribute__(self, field):
         _data = object.__getattribute__(self, '_data')

src/calibre/ebooks/metadata/sources/amazon.py

@@ -16,7 +16,7 @@ from lxml.html import soupparser, tostring
 
 from calibre import as_unicode
 from calibre.ebooks.metadata import check_isbn
-from calibre.ebooks.metadata.sources.base import Source
+from calibre.ebooks.metadata.sources.base import Source, Option
 from calibre.utils.cleantext import clean_ascii_chars
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata.book.base import Metadata
@@ -37,6 +37,92 @@ class Worker(Thread): # Get details {{{
         self.relevance, self.plugin = relevance, plugin
         self.browser = browser.clone_browser()
         self.cover_url = self.amazon_id = self.isbn = None
+        self.domain = self.plugin.domain
+
+        months = {
+                'de': {
+            1 : ['jän'],
+            3 : ['märz'],
+            5 : ['mai'],
+            6 : ['juni'],
+            7 : ['juli'],
+            10: ['okt'],
+            12: ['dez']
+            },
+                'it': {
+            1: ['enn'],
+            2: ['febbr'],
+            5: ['magg'],
+            6: ['giugno'],
+            7: ['luglio'],
+            8: ['ag'],
+            9: ['sett'],
+            10: ['ott'],
+            12: ['dic'],
+            },
+                'fr': {
+            1: ['janv'],
+            2: ['févr'],
+            3: ['mars'],
+            4: ['avril'],
+            5: ['mai'],
+            6: ['juin'],
+            7: ['juil'],
+            8: ['août'],
+            9: ['sept'],
+            12: ['déc'],
+            },
+
+        }
+
+        self.english_months = [None, 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
+                'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+        self.months = months.get(self.domain, {})
+
+        self.pd_xpath = '''
+            //h2[text()="Product Details" or \
+                 text()="Produktinformation" or \
+                 text()="Dettagli prodotto" or \
+                 text()="Product details" or \
+                 text()="Détails sur le produit"]/../div[@class="content"]
+            '''
+        self.publisher_xpath = '''
+            descendant::*[starts-with(text(), "Publisher:") or \
+                    starts-with(text(), "Verlag:") or \
+                    starts-with(text(), "Editore:") or \
+                    starts-with(text(), "Editeur")]
+            '''
+        self.language_xpath =    '''
+            descendant::*[
+                starts-with(text(), "Language:") \
+                or text() = "Language" \
+                or text() = "Sprache:" \
+                or text() = "Lingua:" \
+                or starts-with(text(), "Langue") \
+                ]
+            '''
+        self.ratings_pat = re.compile(
+            r'([0-9.]+) (out of|von|su|étoiles sur) (\d+)( (stars|Sternen|stelle)){0,1}')
+
+        lm = {
+                'en': ('English', 'Englisch'),
+                'fr': ('French', 'Français'),
+                'it': ('Italian', 'Italiano'),
+                'de': ('German', 'Deutsch'),
+                }
+        self.lang_map = {}
+        for code, names in lm.iteritems():
+            for name in names:
+                self.lang_map[name] = code
+
+    def delocalize_datestr(self, raw):
+        if not self.months:
+            return raw
+        ans = raw.lower()
+        for i, vals in self.months.iteritems():
+            for x in vals:
+                ans = ans.replace(x, self.english_months[i])
+        return ans
 
     def run(self):
         try:
@@ -132,7 +218,7 @@ class Worker(Thread): # Get details {{{
             self.log.exception('Error parsing cover for url: %r'%self.url)
         mi.has_cover = bool(self.cover_url)
 
-        pd = root.xpath('//h2[text()="Product Details"]/../div[@class="content"]')
+        pd = root.xpath(self.pd_xpath)
         if pd:
             pd = pd[0]
 
@@ -194,30 +280,42 @@ class Worker(Thread): # Get details {{{
     def parse_authors(self, root):
         x = '//h1[@class="parseasinTitle"]/following-sibling::span/*[(name()="a" and @href) or (name()="span" and @class="contributorNameTrigger")]'
         aname = root.xpath(x)
+        if not aname:
+            aname = root.xpath('''
+            //h1[@class="parseasinTitle"]/following-sibling::*[(name()="a" and @href) or (name()="span" and @class="contributorNameTrigger")]
+                    ''')
         for x in aname:
             x.tail = ''
         authors = [tostring(x, encoding=unicode, method='text').strip() for x
                 in aname]
+        authors = [a for a in authors if a]
         return authors
 
     def parse_rating(self, root):
         ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]')
-        pat = re.compile(r'([0-9.]+) out of (\d+) stars')
+        if not ratings:
+            ratings = root.xpath('//div[@class="buying"]/descendant::span[@class="asinReviewsSummary"]')
+        if not ratings:
+            ratings = root.xpath('//span[@class="crAvgStars"]/descendant::span[@class="asinReviewsSummary"]')
         if ratings:
             for elem in ratings[0].xpath('descendant::*[@title]'):
                 t = elem.get('title').strip()
-                m = pat.match(t)
+                m = self.ratings_pat.match(t)
                 if m is not None:
-                    return float(m.group(1))/float(m.group(2)) * 5
+                    return float(m.group(1))/float(m.group(3)) * 5
 
     def parse_comments(self, root):
         desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]')
         if desc:
             desc = desc[0]
             for c in desc.xpath('descendant::*[@class="seeAll" or'
-                    ' @class="emptyClear" or @href]'):
+                    ' @class="emptyClear"]'):
                 c.getparent().remove(c)
+            for a in desc.xpath('descendant::a[@href]'):
+                del a.attrib['href']
+                a.tag = 'span'
             desc = tostring(desc, method='html', encoding=unicode).strip()
+
             # Encoding bug in Amazon data U+fffd (replacement char)
             # in some examples it is present in place of '
             desc = desc.replace('\ufffd', "'")
@@ -246,41 +344,44 @@ class Worker(Thread): # Get details {{{
                         return ('/'.join(parts[:-1]))+'/'+bn
 
     def parse_isbn(self, pd):
-        for x in reversed(pd.xpath(
-            'descendant::*[starts-with(text(), "ISBN")]')):
+        items = pd.xpath(
+            'descendant::*[starts-with(text(), "ISBN")]')
+        if not items:
+            items = pd.xpath(
+                'descendant::b[contains(text(), "ISBN:")]')
+        for x in reversed(items):
             if x.tail:
                 ans = check_isbn(x.tail.strip())
                 if ans:
                     return ans
 
     def parse_publisher(self, pd):
-        for x in reversed(pd.xpath(
-            'descendant::*[starts-with(text(), "Publisher:")]')):
+        for x in reversed(pd.xpath(self.publisher_xpath)):
             if x.tail:
                 ans = x.tail.partition(';')[0]
                 return ans.partition('(')[0].strip()
 
     def parse_pubdate(self, pd):
-        for x in reversed(pd.xpath(
-            'descendant::*[starts-with(text(), "Publisher:")]')):
+        for x in reversed(pd.xpath(self.publisher_xpath)):
             if x.tail:
                 ans = x.tail
                 date = ans.partition('(')[-1].replace(')', '').strip()
+                date = self.delocalize_datestr(date)
                 return parse_date(date, assume_utc=True)
 
     def parse_language(self, pd):
-        for x in reversed(pd.xpath(
-            'descendant::*[starts-with(text(), "Language:")]')):
+        for x in reversed(pd.xpath(self.language_xpath)):
             if x.tail:
                 ans = x.tail.strip()
-                if ans == 'English':
-                    return 'en'
+                ans = self.lang_map.get(ans, None)
+                if ans:
+                    return ans
 # }}}
 
 class Amazon(Source):
 
     name = 'Amazon.com'
-    description = _('Downloads metadata from Amazon')
+    description = _('Downloads metadata and covers from Amazon')
 
     capabilities = frozenset(['identify', 'cover'])
     touched_fields = frozenset(['title', 'authors', 'identifier:amazon',
@@ -294,8 +395,15 @@ class Amazon(Source):
             'fr' : _('France'),
             'de' : _('Germany'),
             'uk' : _('UK'),
+            'it' : _('Italy'),
     }
 
+    options = (
+            Option('domain', 'choices', 'com', _('Amazon website to use:'),
+                _('Metadata from Amazon will be fetched using this '
+                    'country\'s Amazon website.'), choices=AMAZON_DOMAINS),
+            )
+
     def get_book_url(self, identifiers): # {{{
         asin = identifiers.get('amazon', None)
         if asin is None:
@@ -304,8 +412,16 @@ class Amazon(Source):
             return ('amazon', asin, 'http://amzn.com/%s'%asin)
     # }}}
 
+    @property
+    def domain(self):
+        domain = self.prefs['domain']
+        if domain not in self.AMAZON_DOMAINS:
+            domain = 'com'
+
+        return domain
+
     def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
-        domain = self.prefs.get('domain', 'com')
+        domain = self.domain
 
         # See the amazon detailed search page to get all options
         q = {   'search-alias' : 'aps',
@@ -345,6 +461,8 @@ class Amazon(Source):
         latin1q = dict([(x.encode('latin1', 'ignore'), y.encode('latin1',
             'ignore')) for x, y in
             q.iteritems()])
+        if domain == 'uk':
+            domain = 'co.uk'
         url = 'http://www.amazon.%s/s/?'%domain + urlencode(latin1q)
         return url
 
@@ -516,11 +634,19 @@ if __name__ == '__main__': # tests {{{
     # src/calibre/ebooks/metadata/sources/amazon.py
     from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
             title_test, authors_test)
-    test_identify_plugin(Amazon.name,
-        [
+    com_tests = [ # {{{
 
-            ( # An e-book ISBN not on Amazon, one of the authors is
-              # unknown to Amazon, so no popup wrapper
+            (  # Description has links
+                {'identifiers':{'isbn': '9780671578275'}},
+                [title_test('A Civil Campaign: A Comedy of Biology and Manners',
+                    exact=True), authors_test(['Lois McMaster Bujold'])
+                 ]
+
+            ),
+
+            ( # An e-book ISBN not on Amazon, the title/author search matches
+              # the Kindle edition, which has different markup for ratings and
+              # isbn
                 {'identifiers':{'isbn': '9780307459671'},
                     'title':'Invisible Gorilla', 'authors':['Christopher Chabris']},
                 [title_test('The Invisible Gorilla: And Other Ways Our Intuitions Deceive Us',
@@ -556,6 +682,38 @@ if __name__ == '__main__': # tests {{{
 
             ),
 
-        ])
+    ] # }}}
+
+    de_tests = [ # {{{
+            (
+                {'identifiers':{'isbn': '3548283519'}},
+                [title_test('Wer Wind sät',
+                    exact=True), authors_test(['Nele Neuhaus'])
+                 ]
+
+            ),
+    ] # }}}
+
+    it_tests = [ # {{{
+            (
+                {'identifiers':{'isbn': '8838922195'}},
+                [title_test('La briscola in cinque',
+                    exact=True), authors_test(['Marco Malvaldi'])
+                 ]
+
+            ),
+    ] # }}}
+
+    fr_tests = [ # {{{
+            (
+                {'identifiers':{'isbn': '2221116798'}},
+                [title_test('L\'étrange voyage de Monsieur Daldry',
+                    exact=True), authors_test(['Marc Levy'])
+                 ]
+
+            ),
+    ] # }}}
+
+    test_identify_plugin(Amazon.name, com_tests)
 # }}}
 

src/calibre/ebooks/metadata/sources/base.py

@@ -145,10 +145,13 @@ class Option(object):
         :param default: The default value for this option
         :param label: A short (few words) description of this option
         :param desc: A longer description of this option
-        :param choices: A list of possible values, used only if type='choices'
+        :param choices: A dict of possible values, used only if type='choices'.
+        dict is of the form {key:human readable label, ...}
         '''
         self.name, self.type, self.default, self.label, self.desc = (name,
                 type_, default, label, desc)
+        if choices and not isinstance(choices, dict):
+            choices = dict([(x, x) for x in choices])
         self.choices = choices
 
 class Source(Plugin):

src/calibre/ebooks/metadata/sources/douban.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>; 2011, Li Fanxi <lifanxi@freemindworld.com>'
+__docformat__ = 'restructuredtext en'
+
+import time
+from urllib import urlencode
+from functools import partial
+from Queue import Queue, Empty
+
+from lxml import etree
+
+from calibre.ebooks.metadata import check_isbn
+from calibre.ebooks.metadata.sources.base import Source
+from calibre.ebooks.metadata.book.base import Metadata
+from calibre.ebooks.chardet import xml_to_unicode
+from calibre.utils.date import parse_date, utcnow
+from calibre.utils.cleantext import clean_ascii_chars
+from calibre import as_unicode
+
+NAMESPACES = {
+              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
+              'atom' : 'http://www.w3.org/2005/Atom',
+              'db': 'http://www.douban.com/xmlns/',
+              'gd': 'http://schemas.google.com/g/2005'
+            }
+XPath = partial(etree.XPath, namespaces=NAMESPACES)
+total_results  = XPath('//openSearch:totalResults')
+start_index    = XPath('//openSearch:startIndex')
+items_per_page = XPath('//openSearch:itemsPerPage')
+entry          = XPath('//atom:entry')
+entry_id       = XPath('descendant::atom:id')
+title          = XPath('descendant::atom:title')
+description    = XPath('descendant::atom:summary')
+publisher      = XPath("descendant::db:attribute[@name='publisher']")
+isbn           = XPath("descendant::db:attribute[@name='isbn13']")
+date           = XPath("descendant::db:attribute[@name='pubdate']")
+creator        = XPath("descendant::db:attribute[@name='author']")
+booktag        = XPath("descendant::db:tag/attribute::name")
+rating         = XPath("descendant::gd:rating/attribute::average")
+cover_url      = XPath("descendant::atom:link[@rel='image']/attribute::href")
+
+def get_details(browser, url, timeout): # {{{
+    try:
+        raw = browser.open_novisit(url, timeout=timeout).read()
+    except Exception as e:
+        gc = getattr(e, 'getcode', lambda : -1)
+        if gc() != 403:
+            raise
+        # Douban is throttling us, wait a little
+        time.sleep(2)
+        raw = browser.open_novisit(url, timeout=timeout).read()
+
+    return raw
+# }}}
+
+def to_metadata(browser, log, entry_, timeout): # {{{
+    def get_text(extra, x):
+        try:
+            ans = x(extra)
+            if ans:
+                ans = ans[0].text
+                if ans and ans.strip():
+                    return ans.strip()
+        except:
+            log.exception('Programming error:')
+        return None
+
+    id_url = entry_id(entry_)[0].text
+    douban_id = id_url.split('/')[-1]
+    title_ = ': '.join([x.text for x in title(entry_)]).strip()
+    authors = [x.text.strip() for x in creator(entry_) if x.text]
+    if not authors:
+        authors = [_('Unknown')]
+    if not id_url or not title:
+        # Silently discard this entry
+        return None
+
+    mi = Metadata(title_, authors)
+    mi.identifiers = {'douban':douban_id}
+    try:
+        raw = get_details(browser, id_url, timeout)
+        feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
+            strip_encoding_pats=True)[0])
+        extra = entry(feed)[0]
+    except:
+        log.exception('Failed to get additional details for', mi.title)
+        return mi
+    mi.comments = get_text(extra, description)
+    mi.publisher = get_text(extra, publisher)
+
+    # ISBN
+    isbns = []
+    for x in [t.text for t in isbn(extra)]:
+        if check_isbn(x):
+            isbns.append(x)
+    if isbns:
+        mi.isbn = sorted(isbns, key=len)[-1]
+    mi.all_isbns = isbns
+
+    # Tags
+    try:
+        btags = [x for x in booktag(extra) if x]
+        tags = []
+        for t in btags:
+            atags = [y.strip() for y in t.split('/')]
+            for tag in atags:
+                if tag not in tags:
+                    tags.append(tag)
+    except:
+        log.exception('Failed to parse tags:')
+        tags = []
+    if tags:
+        mi.tags = [x.replace(',', ';') for x in tags]
+
+    # pubdate
+    pubdate = get_text(extra, date)
+    if pubdate:
+        try:
+            default = utcnow().replace(day=15)
+            mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
+        except:
+            log.error('Failed to parse pubdate %r'%pubdate)
+
+    # Ratings
+    if rating(extra):
+        try:
+            mi.rating = float(rating(extra)[0]) / 2.0
+        except:
+            log.exception('Failed to parse rating')
+            mi.rating = 0
+
+    # Cover
+    mi.has_douban_cover = None
+    u = cover_url(extra)
+    if u:
+        u = u[0].replace('/spic/', '/lpic/');
+        # If URL contains "book-default", the book doesn't have a cover
+        if u.find('book-default') == -1:
+            mi.has_douban_cover = u
+    return mi
+# }}}
+
+class Douban(Source):
+
+    name = 'Douban Books'
+    author = 'Li Fanxi'
+    version = (2, 0, 0)
+
+    description = _('Downloads metadata and covers from Douban.com')
+
+    capabilities = frozenset(['identify', 'cover'])
+    touched_fields = frozenset(['title', 'authors', 'tags',
+        'pubdate', 'comments', 'publisher', 'identifier:isbn', 'rating',
+        'identifier:douban']) # language currently disabled
+    supports_gzip_transfer_encoding = True
+    cached_cover_url_is_reliable = True
+
+    DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
+    DOUBAN_BOOK_URL = 'http://book.douban.com/subject/%s/'
+
+    def get_book_url(self, identifiers): # {{{
+        db = identifiers.get('douban', None)
+        if db is not None:
+            return ('douban', db, self.DOUBAN_BOOK_URL%db)
+    # }}}
+
+    def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
+        SEARCH_URL = 'http://api.douban.com/book/subjects?'
+        ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
+        SUBJECT_URL = 'http://api.douban.com/book/subject/'
+
+        q = ''
+        t = None
+        isbn = check_isbn(identifiers.get('isbn', None))
+        subject = identifiers.get('douban', None)
+        if isbn is not None:
+            q = isbn
+            t = 'isbn'
+        elif subject is not None:
+            q = subject
+            t = 'subject'
+        elif title or authors:
+            def build_term(prefix, parts):
+                return ' '.join(x for x in parts)
+            title_tokens = list(self.get_title_tokens(title))
+            if title_tokens:
+                q += build_term('title', title_tokens)
+            author_tokens = self.get_author_tokens(authors,
+                    only_first_author=True)
+            if author_tokens:
+                q += ((' ' if q != '' else '') +
+                    build_term('author', author_tokens))
+            t = 'search'
+        q = q.strip()
+        if isinstance(q, unicode):
+            q = q.encode('utf-8')
+        if not q:
+            return None
+        url = None
+        if t == "isbn":
+            url = ISBN_URL + q
+        elif t == 'subject':
+            url = SUBJECT_URL + q
+        else:
+            url = SEARCH_URL + urlencode({
+                    'q': q,
+                    })
+        if self.DOUBAN_API_KEY and self.DOUBAN_API_KEY != '':
+            url = url + "?apikey=" + self.DOUBAN_API_KEY
+        return url
+    # }}}
+
+    def download_cover(self, log, result_queue, abort, # {{{
+            title=None, authors=None, identifiers={}, timeout=30):
+        cached_url = self.get_cached_cover_url(identifiers)
+        if cached_url is None:
+            log.info('No cached cover found, running identify')
+            rq = Queue()
+            self.identify(log, rq, abort, title=title, authors=authors,
+                    identifiers=identifiers)
+            if abort.is_set():
+                return
+            results = []
+            while True:
+                try:
+                    results.append(rq.get_nowait())
+                except Empty:
+                    break
+            results.sort(key=self.identify_results_keygen(
+                title=title, authors=authors, identifiers=identifiers))
+            for mi in results:
+                cached_url = self.get_cached_cover_url(mi.identifiers)
+                if cached_url is not None:
+                    break
+        if cached_url is None:
+            log.info('No cover found')
+            return
+
+        if abort.is_set():
+            return
+        br = self.browser
+        log('Downloading cover from:', cached_url)
+        try:
+            cdata = br.open_novisit(cached_url, timeout=timeout).read()
+            if cdata:
+                result_queue.put((self, cdata))
+        except:
+            log.exception('Failed to download cover from:', cached_url)
+
+    # }}}
+
+    def get_cached_cover_url(self, identifiers): # {{{
+        url = None
+        db = identifiers.get('douban', None)
+        if db is None:
+            isbn = identifiers.get('isbn', None)
+            if isbn is not None:
+                db = self.cached_isbn_to_identifier(isbn)
+        if db is not None:
+            url = self.cached_identifier_to_cover_url(db)
+
+        return url
+    # }}}
+
+    def get_all_details(self, br, log, entries, abort, # {{{
+            result_queue, timeout):
+        for relevance, i in enumerate(entries):
+            try:
+                ans = to_metadata(br, log, i, timeout)
+                if isinstance(ans, Metadata):
+                    ans.source_relevance = relevance
+                    db = ans.identifiers['douban']
+                    for isbn in getattr(ans, 'all_isbns', []):
+                        self.cache_isbn_to_identifier(isbn, db)
+                    if ans.has_douban_cover:
+                        self.cache_identifier_to_cover_url(db,
+                                ans.has_douban_cover)
+                    self.clean_downloaded_metadata(ans)
+                    result_queue.put(ans)
+            except:
+                log.exception(
+                    'Failed to get metadata for identify entry:',
+                    etree.tostring(i))
+            if abort.is_set():
+                break
+    # }}}
+
+    def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
+            identifiers={}, timeout=30):
+        query = self.create_query(log, title=title, authors=authors,
+                identifiers=identifiers)
+        if not query:
+            log.error('Insufficient metadata to construct query')
+            return
+        br = self.browser
+        try:
+            raw = br.open_novisit(query, timeout=timeout).read()
+        except Exception as e:
+            log.exception('Failed to make identify query: %r'%query)
+            return as_unicode(e)
+        try:
+            parser = etree.XMLParser(recover=True, no_network=True)
+            feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
+                strip_encoding_pats=True)[0], parser=parser)
+            entries = entry(feed)
+        except Exception as e:
+            log.exception('Failed to parse identify results')
+            return as_unicode(e)
+        if not entries and identifiers and title and authors and \
+                not abort.is_set():
+            return self.identify(log, result_queue, abort, title=title,
+                    authors=authors, timeout=timeout)
+
+        # There is no point running these queries in threads as douban
+        # throttles requests returning 403 Forbidden errors
+        self.get_all_details(br, log, entries, abort, result_queue, timeout)
+
+        return None
+    # }}}
+
+if __name__ == '__main__': # tests {{{
+    # To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/douban.py
+    from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
+            title_test, authors_test)
+    test_identify_plugin(Douban.name,
+        [
+
+
+            (
+                {'identifiers':{'isbn': '9787536692930'}, 'title':'三体',
+                    'authors':['刘慈欣']},
+                [title_test('三体', exact=True),
+                    authors_test(['刘慈欣'])]
+            ),
+
+            (
+                {'title': 'Linux内核修炼之道', 'authors':['任桥伟']},
+                [title_test('Linux内核修炼之道', exact=False)]
+            ),
+    ])
+# }}}
+

src/calibre/ebooks/metadata/sources/google.py

@@ -157,7 +157,7 @@ def to_metadata(browser, log, entry_, timeout): # {{{
 class GoogleBooks(Source):
 
     name = 'Google'
-    description = _('Downloads metadata from Google Books')
+    description = _('Downloads metadata and covers from Google Books')
 
     capabilities = frozenset(['identify', 'cover'])
     touched_fields = frozenset(['title', 'authors', 'tags', 'pubdate',

src/calibre/ebooks/metadata/sources/identify.py

@@ -372,6 +372,18 @@ def identify(log, abort, # {{{
     longest, lp = -1, ''
     for plugin, presults in results.iteritems():
         presults.sort(key=plugin.identify_results_keygen(**sort_kwargs))
+
+        # Throw away lower priority results from the same source that have exactly the same
+        # title and authors as a higher priority result
+        filter_results = set()
+        filtered_results = []
+        for r in presults:
+            key = (r.title, tuple(r.authors))
+            if key not in filter_results:
+                filtered_results.append(r)
+                filter_results.add(key)
+        results[plugin] = presults = filtered_results
+
         plog = logs[plugin].getvalue().strip()
         log('\n'+'*'*30, plugin.name, '*'*30)
         log('Request extra headers:', plugin.browser.addheaders)
@@ -479,7 +491,7 @@ if __name__ == '__main__': # tests {{{
             (
                 {'title':'Magykal Papers',
                     'authors':['Sage']},
-                [title_test('The Magykal Papers', exact=True)],
+                [title_test('Septimus Heap: The Magykal Papers', exact=True)],
             ),
 
 
@@ -506,12 +518,6 @@ if __name__ == '__main__': # tests {{{
                     exact=True), authors_test(['Dan Brown'])]
             ),
 
-            ( # No ISBN
-                {'title':'Justine', 'authors':['Durrel']},
-                [title_test('Justine', exact=True),
-                    authors_test(['Lawrence Durrel'])]
-            ),
-
             (  # A newer book
                 {'identifiers':{'isbn': '9780316044981'}},
                 [title_test('The Heroes', exact=True),

src/calibre/ebooks/metadata/sources/overdrive.py

@@ -30,7 +30,7 @@ base_url = 'http://search.overdrive.com/'
 class OverDrive(Source):
 
     name = 'Overdrive'
-    description = _('Downloads metadata from Overdrive\'s Content Reserve')
+    description = _('Downloads metadata and covers from Overdrive\'s Content Reserve')
 
     capabilities = frozenset(['identify', 'cover'])
     touched_fields = frozenset(['title', 'authors', 'tags', 'pubdate',

src/calibre/ebooks/mobi/reader.py

@@ -690,6 +690,14 @@ class MobiReader(object):
                 lm = unit_convert('2em', 12, 500, 166)
             lm = self.left_margins.get(tag, lm)
             ti = self.text_indents.get(tag, ti)
+            try:
+                lm = float(lm)
+            except:
+                lm = 0.0
+            try:
+                ti = float(ti)
+            except:
+                ti = 0.0
             return lm + ti
 
         parent = tag

src/calibre/ebooks/oeb/reader.py

@@ -191,7 +191,11 @@ class OEBReader(object):
                         if not scheme and href not in known:
                             new.add(href)
                 elif item.media_type in OEB_STYLES:
-                    for url in cssutils.getUrls(item.data):
+                    try:
+                        urls = list(cssutils.getUrls(item.data))
+                    except:
+                        urls = []
+                    for url in urls:
                         href, _ = urldefrag(url)
                         href = item.abshref(urlnormalize(href))
                         scheme = urlparse(href).scheme

src/calibre/ebooks/pdf/images.cpp

@@ -13,6 +13,7 @@
 #include <math.h>
 #include <iostream>
 #include <wand/MagickWand.h>
+#include <zlib.h>
 
 #include "images.h"
 #include "utils.h"

src/calibre/ebooks/rtf/input.py

@@ -86,7 +86,7 @@ class RTFInput(InputFormatPlugin):
                 run_lev = 4
                 self.log('Running RTFParser in debug mode')
             except:
-                pass
+                self.log.warn('Impossible to run RTFParser in debug mode')
         parser = ParseRtf(
             in_file    = stream,
             out_file   = ofile,

src/calibre/ebooks/rtf2xml/process_tokens.py

@@ -197,8 +197,8 @@ class ProcessTokens:
         # character info => ci
         'b'                  :	('ci', 'bold______', self.bool_st_func),
         'blue'               :	('ci', 'blue______', self.color_func),
-        'caps'               :  ('ci', 'caps______', self.bool_st_func),
-        'cf'                 :	('ci', 'font-color', self.default_func),
+        'caps'               :	('ci', 'caps______', self.bool_st_func),
+        'cf'                 :	('ci', 'font-color', self.colorz_func),
         'chftn'              :	('ci', 'footnot-mk', self.bool_st_func),
         'dn'                 :	('ci', 'font-down_', self.divide_by_2),
         'embo'               :	('ci', 'emboss____', self.bool_st_func),
@@ -624,6 +624,11 @@ class ProcessTokens:
             num = 'true'
         return 'cw<%s<%s<nu<%s\n' % (pre, token, num)
 
+    def colorz_func(self, pre, token, num):
+        if num is None:
+            num = '0'
+        return 'cw<%s<%s<nu<%s\n' % (pre, token, num)
+
     def __list_type_func(self, pre, token, num):
         type = 'arabic'
         if num is None:

src/calibre/ebooks/textile/functions.py

@@ -12,7 +12,7 @@ A Humane Web Text Generator
 #__date__ = '2009/12/04'
 
 __copyright__ = """
-Copyright (c) 2011, Leigh Parry
+Copyright (c) 2011, Leigh Parry <leighparry@blueyonder.co.uk>
 Copyright (c) 2011, John Schember <john@nachtimwald.com>
 Copyright (c) 2009, Jason Samsa, http://jsamsa.com/
 Copyright (c) 2004, Roberto A. F. De Almeida, http://dealmeida.net/
@@ -219,14 +219,13 @@ class Textile(object):
     ]
     glyph_defaults = [
         (re.compile(r'(\d+\'?\"?)( ?)x( ?)(?=\d+)'),                   r'\1\2&#215;\3'),                       #  dimension sign
-        (re.compile(r'(\d+)\'', re.I),                                 r'\1&#8242;'),                          #  prime
-        (re.compile(r'(\d+)\"', re.I),                                 r'\1&#8243;'),                          #  prime-double
+        (re.compile(r'(\d+)\'(\s)', re.I),                             r'\1&#8242;\2'),                          #  prime
+        (re.compile(r'(\d+)\"(\s)', re.I),                             r'\1&#8243;\2'),                          #  prime-double
         (re.compile(r'\b([A-Z][A-Z0-9]{2,})\b(?:[(]([^)]*)[)])'),      r'<acronym title="\2">\1</acronym>'),   #  3+ uppercase acronym
         (re.compile(r'\b([A-Z][A-Z\'\-]+[A-Z])(?=[\s.,\)>])'),         r'<span class="caps">\1</span>'),       #  3+ uppercase
         (re.compile(r'\b(\s{0,1})?\.{3}'),                             r'\1&#8230;'),                          #  ellipsis
         (re.compile(r'^[\*_-]{3,}$', re.M),                            r'<hr />'),                             #  <hr> scene-break
-        (re.compile(r'\b--\b'),                                        r'&#8212;'),                            #  em dash
-        (re.compile(r'(\s)--(\s)'),                                    r'\1&#8212;\2'),                        #  em dash
+        (re.compile(r'(^|[^-])--([^-]|$)'),                                r'\1&#8212;\2'),                        #  em dash
         (re.compile(r'\s-(?:\s|$)'),                                   r' &#8211; '),                          #  en dash
         (re.compile(r'\b( ?)[([]TM[])]', re.I),                        r'\1&#8482;'),                          #  trademark
         (re.compile(r'\b( ?)[([]R[])]', re.I),                         r'\1&#174;'),                           #  registered
@@ -706,6 +705,21 @@ class Textile(object):
             result.append(line)
         return ''.join(result)
 
+    def macros_only(self, text):
+        # fix: hackish
+        text = re.sub(r'"\Z', '\" ', text)
+
+        result = []
+        for line in re.compile(r'(<.*?>)', re.U).split(text):
+            if not re.search(r'<.*>', line):
+                rules = []
+                if re.search(r'{.+?}', line):
+                    rules = self.macro_defaults
+                for s, r in rules:
+                    line = s.sub(r, line)
+            result.append(line)
+        return ''.join(result)
+
     def vAlign(self, input):
         d = {'^':'top', '-':'middle', '~':'bottom'}
         return d.get(input, '')
@@ -814,6 +828,7 @@ class Textile(object):
         'fooobar ... and hello world ...'
         """
 
+        text = self.macros_only(text)
         punct = '!"#$%&\'*+,-./:;=?@\\^_`|~'
 
         pattern = r'''
@@ -1044,4 +1059,3 @@ def textile_restricted(text, lite=True, noimage=True, html_type='xhtml'):
     return Textile(restricted=True, lite=lite,
                    noimage=noimage).textile(text, rel='nofollow',
                                             html_type=html_type)
-

src/calibre/ebooks/txt/output.py

@@ -66,19 +66,26 @@ class TXTOutput(OutputFormatPlugin):
             help=_('Do not remove image references within the document. This is only ' \
             'useful when paired with a txt-output-formatting option that '
             'is not none because links are always removed with plain text output.')),
+        OptionRecommendation(name='keep_color',
+            recommended_value=False, level=OptionRecommendation.LOW,
+            help=_('Do not remove font color from output. This is only useful when ' \
+                   'txt-output-formatting is set to textile. Textile is the only ' \
+                   'formatting that supports setting font color. If this option is ' \
+                   'not specified font color will not be set and default to the ' \
+                   'color displayed by the reader (generally this is black).')),
      ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
         if opts.txt_output_formatting.lower() == 'markdown':
             from calibre.ebooks.txt.markdownml import MarkdownMLizer
-            writer = MarkdownMLizer(log)
+            self.writer = MarkdownMLizer(log)
         elif opts.txt_output_formatting.lower() == 'textile':
             from calibre.ebooks.txt.textileml import TextileMLizer
-            writer = TextileMLizer(log)
+            self.writer = TextileMLizer(log)
         else:
-            writer = TXTMLizer(log)
+            self.writer = TXTMLizer(log)
 
-        txt = writer.extract_content(oeb_book, opts)
+        txt = self.writer.extract_content(oeb_book, opts)
         txt = clean_ascii_chars(txt)
 
         log.debug('\tReplacing newlines with selected type...')
@@ -111,17 +118,28 @@ class TXTZOutput(TXTOutput):
         from calibre.ebooks.oeb.base import OEB_IMAGES
         with TemporaryDirectory('_txtz_output') as tdir:
             # TXT
-            with TemporaryFile('index.txt') as tf:
+            txt_name = 'index.txt'
+            if opts.txt_output_formatting.lower() == 'textile':
+                txt_name = 'index.text'
+            with TemporaryFile(txt_name) as tf:
                 TXTOutput.convert(self, oeb_book, tf, input_plugin, opts, log)
-                shutil.copy(tf, os.path.join(tdir, 'index.txt'))
+                shutil.copy(tf, os.path.join(tdir, txt_name))
 
             # Images
             for item in oeb_book.manifest:
                 if item.media_type in OEB_IMAGES:
-                    path = os.path.join(tdir, os.path.dirname(item.href))
+                    if hasattr(self.writer, 'images'):
+                        path = os.path.join(tdir, 'images')
+                        if item.href in self.writer.images:
+                            href = self.writer.images[item.href]
+                        else:
+                            continue
+                    else:
+                        path = os.path.join(tdir, os.path.dirname(item.href))
+                        href = os.path.basename(item.href)
                     if not os.path.exists(path):
                         os.makedirs(path)
-                    with open(os.path.join(tdir, item.href), 'wb') as imgf:
+                    with open(os.path.join(path, href), 'wb') as imgf:
                         imgf.write(item.data)
 
             # Metadata

src/calibre/ebooks/txt/processor.py

@@ -242,6 +242,8 @@ def detect_formatting_type(txt):
     textile_count += len(re.findall(r'(?mu)(?<=\!)\S+(?=\!)', txt))
     # Links
     textile_count += len(re.findall(r'"[^"]*":\S+', txt))
+    # paragraph blocks
+    textile_count += len(re.findall(r'(?mu)^p(<|<>|=|>)?\. ', txt))
 
     # Decide if either markdown or textile is used in the text
     # based on the number of unique formatting elements found.

src/calibre/ebooks/txt/textileml.py

@@ -1,62 +1,489 @@
 # -*- coding: utf-8 -*-
 
 __license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__copyright__ = '2011, Leigh Parry <leighparry@blueyonder.co.uk>'
 __docformat__ = 'restructuredtext en'
 
 '''
 Transform OEB content into Textile formatted plain text
 '''
-
 import re
 
-from lxml import etree
+from functools import partial
 
-from calibre.ebooks.oeb.base import XHTML
-from calibre.utils.html2textile import html2textile
+from calibre.ebooks.htmlz.oeb2html import OEB2HTML
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, rewrite_links
+from calibre.ebooks.oeb.stylizer import Stylizer
+from calibre.ebooks import unit_convert
+from calibre.ebooks.txt.unsmarten import unsmarten
 
-class TextileMLizer(object):
-
-    def __init__(self, log):
-        self.log = log
+class TextileMLizer(OEB2HTML):
 
     def extract_content(self, oeb_book, opts):
         self.log.info('Converting XHTML to Textile formatted TXT...')
-        self.oeb_book = oeb_book
         self.opts = opts
+        self.in_pre = False
+        self.in_table = False
+        self.links = {}
+        self.list = []
+        self.our_links = []
+        self.in_a_link = False
+        self.our_ids = []
+        self.images = {}
+        self.id_no_text = u''
+        self.style_embed = []
+        self.remove_space_after_newline = False
+        self.base_hrefs = [item.href for item in oeb_book.spine]
+        self.map_resources(oeb_book)
 
-        return self.mlize_spine()
+        self.style_bold = False
+        self.style_italic = False
+        self.style_under = False
+        self.style_strike = False
+        self.style_smallcap = False
 
-    def mlize_spine(self):
+        txt = self.mlize_spine(oeb_book)
+        txt = unsmarten(txt)
+
+        # Do some tidying up
+        txt = self.tidy_up(txt)
+
+        return txt
+
+    def mlize_spine(self, oeb_book):
         output = [u'']
-
-        for item in self.oeb_book.spine:
+        for item in oeb_book.spine:
             self.log.debug('Converting %s to Textile formatted TXT...' % item.href)
+            self.rewrite_ids(item.data, item)
+            rewrite_links(item.data, partial(self.rewrite_link, page=item))
+            stylizer = Stylizer(item.data, item.href, oeb_book, self.opts, self.opts.output_profile)
+            output += self.dump_text(item.data.find(XHTML('body')), stylizer)
+            output.append('\n\n')
+        return ''.join(output)
 
-            html = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode))
+    def tidy_up(self, text):
+        # May need tweaking and finetuning
+        def check_escaping(text, tests):
+            for t in tests:
+                # I'm not checking for duplicated spans '%' as any that follow each other were being incorrectly merged
+                txt = '%s' % t
+                if txt != '%':
+                    text = re.sub(r'([^'+t+'|^\n])'+t+'\]\['+t+'([^'+t+'])', r'\1\2', text)
+                    text = re.sub(r'([^'+t+'|^\n])'+t+t+'([^'+t+'])', r'\1\2', text)
+                text = re.sub(r'(\s|[*_\'"])\[('+t+'[a-zA-Z0-9 \'",.*_]+'+t+')\](\s|[*_\'"?!,.])', r'\1\2\3', text)
+            return text
 
-            if not self.opts.keep_links:
-                html = re.sub(r'<\s*/*\s*a[^>]*>', '', html)
-            if not self.opts.keep_image_references:
-                html = re.sub(r'<\s*img[^>]*>', '', html)
+        # Now tidyup links and ids - remove ones that don't have a correponding opposite
+        if self.opts.keep_links:
+            for i in self.our_links:
+                if i[0] == '#':
+                    if i not in self.our_ids:
+                        text = re.sub(r'"(.+)":'+i+'(\s)', r'\1\2', text)
+            for i in self.our_ids:
+                if i not in self.our_links:
+                    text = re.sub(r'%?\('+i+'\)\xa0?%?', r'', text)
                     
-            text = html2textile(html)
+        # Remove obvious non-needed escaping, add sub/sup-script ones
+        text = check_escaping(text, ['\*', '_', '\*'])
+        # escape the super/sub-scripts if needed
+        text = re.sub(r'(\w)([~^]\w+[~^])', r'\1[\2]', text)
+        # escape the super/sub-scripts if needed
+        text = re.sub(r'([~^]\w+[~^])(\w)', r'[\1]\2', text)
 
-            # Ensure the section ends with at least two new line characters.
-            # This is to prevent the last paragraph from a section being
-            # combined into the fist paragraph of the next.
-            end_chars = text[-4:]
-            # Convert all newlines to \n
-            end_chars = end_chars.replace('\r\n', '\n')
-            end_chars = end_chars.replace('\r', '\n')
-            end_chars = end_chars[-2:]
-            if not end_chars[1] == '\n':
-                text += '\n\n'
-            if end_chars[1] == '\n' and not end_chars[0] == '\n':
-                text += '\n'
+        #remove empty spans
+        text = re.sub(r'%\xa0+', r'%', text)
+        #remove empty spans - MAY MERGE SOME ?
+        text = re.sub(r'%%', r'', text)
+        #remove spans from tagged output
+        text = re.sub(r'%([_+*-]+)%', r'\1', text)
+        #remove spaces before a newline
+        text = re.sub(r' +\n', r'\n', text)
+        #remove newlines at top of file
+        text = re.sub(r'^\n+', r'', text)
+        #correct blockcode paras
+        text = re.sub(r'\npre\.\n?\nbc\.', r'\nbc.', text)
+        #correct blockquote paras
+        text = re.sub(r'\nbq\.\n?\np.*\. ', r'\nbq. ', text)
 
-            output += text
+        #reduce blank lines
+        text = re.sub(r'\n{3}', r'\n\np. \n\n', text)
+        text = re.sub(u'%\n(p[<>=]{1,2}\.|p\.)', r'%\n\n\1', text)
+        #Check span following blank para
+        text = re.sub(r'\n+ +%', r' %', text)
+        text = re.sub(u'p[<>=]{1,2}\.\n\n?', r'', text)
+        # blank paragraph
+        text = re.sub(r'\n(p.*\.)\n', r'\n\1 \n\n', text)
+        # blank paragraph
+        text = re.sub(u'\n\xa0', r'\np. ', text)
+        # blank paragraph
+        text = re.sub(u'\np[<>=]{1,2}?\. \xa0', r'\np. ', text)
+        text = re.sub(r'(^|\n)(p.*\. ?\n)(p.*\.)', r'\1\3', text)
+        text = re.sub(r'\n(p\. \n)(p.*\.|h.*\.)', r'\n\2', text)
+        #sort out spaces in tables
+        text = re.sub(r' {2,}\|', r' |', text)
 
-        output = u''.join(output)
+        # Now put back spaces removed earlier as they're needed here
+        text = re.sub(r'\np\.\n', r'\np. \n', text)
+        #reduce blank lines
+        text = re.sub(r' \n\n\n', r' \n\n', text)
 
-        return output
+        return text
+
+    def remove_newlines(self, text):
+        text = text.replace('\r\n', ' ')
+        text = text.replace('\n', ' ')
+        text = text.replace('\r', ' ')
+        # Condense redundant spaces created by replacing newlines with spaces.
+        text = re.sub(r'[ ]{2,}', ' ', text)
+        text = re.sub(r'\t+', '', text)
+        if self.remove_space_after_newline == True:
+            text = re.sub(r'^ +', '', text)
+            self.remove_space_after_newline = False
+        return text
+
+    def check_styles(self, style):
+        txt = '{'
+        if self.opts.keep_color:
+            if 'color' in style.cssdict() and style['color'] != 'black':
+                txt += 'color:'+style['color']+';'
+            if 'background' in style.cssdict():
+                txt += 'background:'+style['background']+';'
+        txt += '}'
+        if txt == '{}': txt = ''
+        return txt
+
+    def check_halign(self, style):
+        tests = {'left':'<','justify':'<>','center':'=','right':'>'}
+        for i in tests:
+            if style['text-align'] == i:
+                return tests[i]
+        return ''
+
+    def check_valign(self, style):
+        tests = {'top':'^','bottom':'~'} #, 'middle':'-'}
+        for i in tests:
+            if style['vertical-align'] == i:
+                return tests[i]
+        return ''
+
+    def check_padding(self, style, stylizer):
+        txt = ''
+        left_padding_pts = 0
+        left_margin_pts = 0
+        if 'padding-left' in style.cssdict() and style['padding-left'] != 'auto':
+            left_padding_pts = unit_convert(style['padding-left'], style.width, style.fontSize, stylizer.profile.dpi)
+        if 'margin-left' in style.cssdict() and style['margin-left'] != 'auto':
+            left_margin_pts = unit_convert(style['margin-left'], style.width, style.fontSize, stylizer.profile.dpi)
+        left = left_margin_pts + left_padding_pts
+        emleft = int(round(left / stylizer.profile.fbase))
+        if emleft >= 1:
+            txt += '(' * emleft
+        right_padding_pts = 0
+        right_margin_pts = 0
+        if 'padding-right' in style.cssdict() and style['padding-right'] != 'auto':
+            right_padding_pts = unit_convert(style['padding-right'], style.width, style.fontSize, stylizer.profile.dpi)
+        if 'margin-right' in style.cssdict() and style['margin-right'] != 'auto':
+            right_margin_pts = unit_convert(style['margin-right'], style.width, style.fontSize, stylizer.profile.dpi)
+        right = right_margin_pts + right_padding_pts
+        emright = int(round(right / stylizer.profile.fbase))
+        if emright >= 1:
+            txt += ')' * emright
+            
+        return txt
+
+    def check_id_tag(self, attribs):
+        txt = ''
+        if attribs.has_key('id'):
+            txt = '(#'+attribs['id']+ ')'
+            self.our_ids.append('#'+attribs['id'])
+            self.id_no_text = u'\xa0'
+        return txt
+
+    def build_block(self, tag, style, attribs, stylizer):
+        txt = '\n' + tag
+        if self.opts.keep_links:
+            txt += self.check_id_tag(attribs)
+        txt += self.check_padding(style, stylizer)
+        txt += self.check_halign(style)
+        txt += self.check_styles(style)
+        return txt
+
+    def prepare_string_for_textile(self, txt):
+        if re.search(r'(\s([*&_+\-~@%|]|\?{2})\S)|(\S([*&_+\-~@%|]|\?{2})\s)', txt):
+            return ' ==%s== ' % txt
+        return txt
+
+    def dump_text(self, elem, stylizer):
+        '''
+        @elem: The element in the etree that we are working on.
+        @stylizer: The style information attached to the element.
+        '''
+
+        # We can only processes tags. If there isn't a tag return any text.
+        if not isinstance(elem.tag, basestring) \
+           or namespace(elem.tag) != XHTML_NS:
+            p = elem.getparent()
+            if p is not None and isinstance(p.tag, basestring) and namespace(p.tag) == XHTML_NS \
+                    and elem.tail:
+                return [elem.tail]
+            return ['']
+
+        # Setup our variables.
+        text = ['']
+        style = stylizer.style(elem)
+        tags = []
+        tag = barename(elem.tag)
+        attribs = elem.attrib
+        
+        # Ignore anything that is set to not be displayed.
+        if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \
+           or style['visibility'] == 'hidden':
+            return ['']
+
+        # Soft scene breaks.
+        if 'margin-top' in style.cssdict() and style['margin-top'] != 'auto':
+            ems = int(round(float(style.marginTop) / style.fontSize) - 1)
+            if ems >= 1:
+                text.append(u'\n\n\xa0' * ems)
+            
+        if tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'div'):
+            if tag == 'div':
+                tag = 'p'
+            text.append(self.build_block(tag, style, attribs, stylizer))
+            text.append('. ')
+            tags.append('\n')
+
+        if style['font-style'] == 'italic' or tag in ('i', 'em'):
+            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'cite'):
+                if self.style_italic == False:
+                    if self.in_a_link:
+                        text.append('_')
+                        tags.append('_')
+                    else:
+                        text.append('[_')
+                        tags.append('_]')
+                    self.style_embed.append('_')
+                    self.style_italic = True
+        if style['font-weight'] in ('bold', 'bolder') or tag in ('b', 'strong'):
+            if tag not in ('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'th'):
+                if self.style_bold == False:
+                    if self.in_a_link:
+                        text.append('*')
+                        tags.append('*')
+                    else:
+                        text.append('[*')
+                        tags.append('*]')
+                    self.style_embed.append('*')
+                    self.style_bold = True
+        if style['text-decoration'] == 'underline' or tag in ('u', 'ins'):
+            if tag != 'a':
+                if self.style_under == False:
+                    text.append('[+')
+                    tags.append('+]')
+                    self.style_embed.append('+')
+                    self.style_under = True
+        if style['text-decoration'] == 'line-through' or tag in ('strike', 'del', 's'):
+            if self.style_strike == False:
+                text.append('[-')
+                tags.append('-]')
+                self.style_embed.append('-')
+                self.style_strike = True
+        if tag == 'br':
+            for i in reversed(self.style_embed):
+                text.append(i)
+            text.append('\n')
+            for i in self.style_embed:
+                text.append(i)
+            tags.append('')
+            self.remove_space_after_newline = True
+        if tag == 'blockquote':
+            text.append('\nbq. ')
+            tags.append('\n')
+        elif tag in ('abbr', 'acronym'):
+            text.append('')
+            txt = attribs['title']
+            tags.append('(' + txt + ')')
+        elif tag == 'sup':
+            text.append('^')
+            tags.append('^')
+        elif tag == 'sub':
+            text.append('~')
+            tags.append('~')
+        elif tag == 'code':
+            if self.in_pre:
+                text.append('\nbc. ')
+                tags.append('')
+            else:
+                text.append('@')
+                tags.append('@')
+        elif tag == 'cite':
+            text.append('??')
+            tags.append('??')
+        elif tag == 'hr':
+            text.append('\n***')
+            tags.append('\n')
+        elif tag == 'pre':
+            self.in_pre = True
+            text.append('\npre. ')
+            tags.append('pre\n')
+        elif tag == 'a':
+            if self.opts.keep_links:
+                if attribs.has_key('href'):
+                    text.append('"')
+                    tags.append('a')
+                    tags.append('":' + attribs['href'])
+                    self.our_links.append(attribs['href'])
+                    if attribs.has_key('title'):
+                        tags.append('(' + attribs['title'] + ')')
+                    self.in_a_link = True
+                else:
+                    text.append('%')
+                    tags.append('%')
+        elif tag == 'img':
+            if self.opts.keep_image_references:
+                txt = '!' + self.check_halign(style)
+                txt += self.check_valign(style)
+                txt += attribs['src']
+                text.append(txt)
+                if attribs.has_key('alt'):
+                    txt = attribs['alt']
+                    if txt != '':
+                        text.append('(' + txt + ')')
+                tags.append('!')
+        elif tag in ('ol', 'ul'):
+            self.list.append({'name': tag, 'num': 0})
+            text.append('')
+            tags.append(tag)
+        elif tag == 'li':
+            if self.list: li = self.list[-1]
+            else: li = {'name': 'ul', 'num': 0}
+            text.append('\n')
+            if li['name'] == 'ul':
+                text.append('*' * len(self.list) + ' ')
+            elif li['name'] == 'ol':
+                text.append('#' * len(self.list) + ' ')
+            tags.append('')
+        elif tag == 'dl':
+            text.append('\n')
+            tags.append('')
+        elif tag == 'dt':
+            text.append('')
+            tags.append('\n')
+        elif tag == 'dd':
+            text.append('    ')
+            tags.append('')
+        elif tag == 'dd':
+            text.append('')
+            tags.append('\n')
+        elif tag == 'table':
+            txt = self.build_block(tag, style, attribs, stylizer)
+            txt += '. \n'
+            if txt != '\ntable. \n':
+                text.append(txt)
+            else:
+                text.append('\n')
+            tags.append('')
+        elif tag == 'tr':
+            txt = self.build_block('', style, attribs, stylizer)
+            txt += '. '
+            if txt != '\n. ':
+                txt = re.sub ('\n', '', txt)
+                text.append(txt)
+            tags.append('|\n')
+        elif tag == 'td':
+            text.append('|')
+            txt = ''
+            txt += self.check_halign(style)
+            txt += self.check_valign(style)
+            if attribs.has_key ('colspan'):
+                txt += '\\' + attribs['colspan']
+            if attribs.has_key ('rowspan'):
+                txt += '/' + attribs['rowspan']
+            txt += self.check_styles(style)
+            if txt != '':
+                text.append(txt + '. ')
+            tags.append('')
+        elif tag == 'th':
+            text.append('|_. ')
+            tags.append('')
+        elif tag == 'span':
+            if style['font-variant'] == 'small-caps':
+                if self.style_smallcap == False:
+                    text.append('&')
+                    tags.append('&')
+                    self.style_smallcap = True
+            else:
+                if self.in_a_link == False:
+                    txt = '%'
+                    if self.opts.keep_links:
+                        txt += self.check_id_tag(attribs)
+                        txt += self.check_styles(style)
+                    if txt != '%':
+                        text.append(txt)
+                        tags.append('%')
+
+        if self.opts.keep_links and attribs.has_key('id'):
+            if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'span', 'table'):
+                text.append(self.check_id_tag(attribs))
+
+        # Process the styles for any that we want to keep
+        if tag not in ('body', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'hr', 'a', 'img', \
+                'span', 'table', 'tr', 'td'):
+            if not self.in_a_link:
+                text.append(self.check_styles(style))
+        
+        # Process tags that contain text.
+        if hasattr(elem, 'text') and elem.text:
+            txt = elem.text
+            if not self.in_pre:
+                txt = self.prepare_string_for_textile(self.remove_newlines(txt))
+            text.append(txt)
+            self.id_no_text = u''
+
+        # Recurse down into tags within the tag we are in.
+        for item in elem:
+            text += self.dump_text(item, stylizer)
+
+        # Close all open tags.
+        tags.reverse()
+        for t in tags:
+            if tag in ('pre', 'ul', 'ol', 'li', 'table'):
+                if tag == 'pre':
+                    self.in_pre = False
+                elif tag in ('ul', 'ol'):
+                    if self.list: self.list.pop()
+                    if not self.list: text.append('\n')
+            else:
+                if t == 'a':
+                    self.in_a_link = False
+                    t = ''
+                text.append(self.id_no_text)
+                self.id_no_text = u''
+                if t in ('*]', '*'):
+                    self.style_bold = False
+                elif t in ('_]', '_'):
+                    self.style_italic = False
+                elif t == '+]':
+                    self.style_under = False
+                elif t == '-]':
+                    self.style_strike = False
+                elif t == '&':
+                    self.style_smallcap = False
+                if t in ('*]', '_]', '+]', '-]', '*', '_'):
+                    txt = self.style_embed.pop()
+                text.append('%s' % t)
+
+        # Soft scene breaks.
+        if 'margin-bottom' in style.cssdict() and style['margin-bottom'] != 'auto':
+            ems = int(round((float(style.marginBottom) / style.fontSize) - 1))
+            if ems >= 1:
+                text.append(u'\n\n\xa0' * ems)
+
+        # Add the text that is outside of the tag.
+        if hasattr(elem, 'tail') and elem.tail:
+            tail = elem.tail
+            if not self.in_pre:
+                tail = self.prepare_string_for_textile(self.remove_newlines(tail))
+            text.append(tail)
+
+        return text

src/calibre/ebooks/txt/unsmarten.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+"""unsmarten : html2textile helper function"""
+
+__version__ = '0.1'
+__author__ = 'Leigh Parry'
+
+import re
+
+def unsmarten(txt):
+    txt = re.sub(u'–|–|–', r'-', txt) # en-dash
+    txt = re.sub(u'—|—|—', r'--', txt) # em-dash
+    txt = re.sub(u'…|…|…', r'...', txt) # ellipsis
+
+    txt = re.sub(u'“|”|″|“|”|″|“|”|″', r'"', txt)  # double quote
+    txt = re.sub(u'(["\'‘“]|\s)’', r"\1{'/}", txt)  # apostrophe
+    txt = re.sub(u'‘|’|′|‘|’|′|‘|’|′', r"'", txt)  # single quote
+
+    txt = re.sub(u'¢|¢|¢',     r'{c\}',  txt)  # cent
+    txt = re.sub(u'£|£|£',    r'{L-}',  txt)  # pound
+    txt = re.sub(u'¥|¥|¥',      r'{Y=}',  txt)  # yen
+    txt = re.sub(u'©|©|©',     r'{(c)}', txt)  # copyright
+    txt = re.sub(u'®|®|®',      r'{(r)}', txt)  # registered
+    txt = re.sub(u'¼|¼|¼',   r'{1/4}', txt)  # quarter
+    txt = re.sub(u'½|½|½',   r'{1/2}', txt)  # half
+    txt = re.sub(u'¾|¾|¾',   r'{3/4}', txt)  # three-quarter
+    txt = re.sub(u'À|À|À',   r'{A`)}', txt)  # A-grave
+    txt = re.sub(u'Á|Á|Á',   r"{A'}",  txt)  # A-acute
+    txt = re.sub(u'Â|Â|Â',    r'{A^}', txt)  # A-circumflex
+    txt = re.sub(u'Ã|Ã|Ã',   r'{A~}',  txt)  # A-tilde
+    txt = re.sub(u'Ä|Ä|Ä',     r'{A"}',  txt)  # A-umlaut
+    txt = re.sub(u'Å|Å|Å',    r'{Ao}',  txt)  # A-ring
+    txt = re.sub(u'Æ|Æ|Æ',    r'{AE}',  txt)  # AE
+    txt = re.sub(u'Ç|Ç|Ç',   r'{C,}',  txt)  # C-cedilla
+    txt = re.sub(u'È|È|È',   r'{E`}',  txt)  # E-grave
+    txt = re.sub(u'É|É|É',   r"{E'}",  txt)  # E-acute
+    txt = re.sub(u'Ê|Ê|Ê',    r'{E^}', txt)  # E-circumflex
+    txt = re.sub(u'Ë|Ë|Ë',     r'{E"}',  txt)  # E-umlaut
+    txt = re.sub(u'Ì|Ì|Ì',   r'{I`}',  txt)  # I-grave
+    txt = re.sub(u'Í|Í|Í',   r"{I'}",  txt)  # I-acute
+    txt = re.sub(u'Î|Î|Î',    r'{I^}', txt)  # I-circumflex
+    txt = re.sub(u'Ï|Ï|Ï',     r'{I"}',  txt)  # I-umlaut
+    txt = re.sub(u'Ð|Ð|Ð',      r'{D-}',  txt)  # ETH
+    txt = re.sub(u'Ñ|Ñ|Ñ',   r'{N~}',  txt)  # N-tilde
+    txt = re.sub(u'Ò|Ò|Ò',   r'{O`}',  txt)  # O-grave
+    txt = re.sub(u'Ó|Ó|Ó',   r"{O'}",  txt)  # O-acute
+    txt = re.sub(u'Ô|Ô|Ô',    r'{O^}', txt)  # O-circumflex
+    txt = re.sub(u'Õ|Õ|Õ',   r'{O~}',  txt)  # O-tilde
+    txt = re.sub(u'Ö|Ö|Ö',     r'{O"}',  txt)  # O-umlaut
+    txt = re.sub(u'×|×|×',    r'{x}',   txt)  # dimension
+    txt = re.sub(u'Ø|Ø|Ø',   r'{O/}',  txt)  # O-slash
+    txt = re.sub(u'Ù|Ù|Ù',   r"{U`}",  txt)  # U-grave
+    txt = re.sub(u'Ú|Ú|Ú',   r"{U'}",  txt)  # U-acute
+    txt = re.sub(u'Û|Û|Û',    r'{U^}', txt)  # U-circumflex
+    txt = re.sub(u'Ü|Ü|Ü',     r'{U"}',  txt)  # U-umlaut
+    txt = re.sub(u'Ý|Ý|Ý',   r"{Y'}",  txt)  # Y-grave
+    txt = re.sub(u'ß|ß|ß',    r'{sz}',  txt)  # sharp-s
+    txt = re.sub(u'à|à|à',   r'{a`}',  txt)  # a-grave
+    txt = re.sub(u'á|á|á',   r"{a'}",  txt)  # a-acute
+    txt = re.sub(u'â|â|â',    r'{a^}', txt)  # a-circumflex
+    txt = re.sub(u'ã|ã|ã',   r'{a~}',  txt)  # a-tilde
+    txt = re.sub(u'ä|ä|ä',     r'{a"}',  txt)  # a-umlaut
+    txt = re.sub(u'å|å|å',    r'{ao}',  txt)  # a-ring
+    txt = re.sub(u'æ|æ|æ',    r'{ae}',  txt)  # ae
+    txt = re.sub(u'ç|ç|ç',   r'{c,}',  txt)  # c-cedilla
+    txt = re.sub(u'è|è|è',   r'{e`}',  txt)  # e-grave
+    txt = re.sub(u'é|é|é',   r"{e'}",  txt)  # e-acute
+    txt = re.sub(u'ê|ê|ê',    r'{e^}', txt)  # e-circumflex
+    txt = re.sub(u'ë|ë|ë',     r'{e"}',  txt)  # e-umlaut
+    txt = re.sub(u'ì|ì|ì',   r'{i`}',  txt)  # i-grave
+    txt = re.sub(u'í|í|í',   r"{i'}",  txt)  # i-acute
+    txt = re.sub(u'î|î|î',    r'{i^}', txt)  # i-circumflex
+    txt = re.sub(u'ï|ï|ï',     r'{i"}',  txt)  # i-umlaut
+    txt = re.sub(u'ð|ð|ð',      r'{d-}',  txt)  # eth
+    txt = re.sub(u'ñ|ñ|ñ',   r'{n~}',  txt)  # n-tilde
+    txt = re.sub(u'ò|ò|ò',   r'{o`}',  txt)  # o-grave
+    txt = re.sub(u'ó|ó|ó',   r"{o'}",  txt)  # o-acute
+    txt = re.sub(u'ô|ô|ô',    r'{o^}', txt)  # o-circumflex
+    txt = re.sub(u'õ|õ|õ',   r'{o~}',  txt)  # o-tilde
+    txt = re.sub(u'ö|ö|ö',     r'{o"}',  txt)  # o-umlaut
+    txt = re.sub(u'ø|ø|ø',   r'{o/}',  txt)  # o-stroke
+    txt = re.sub(u'ù|ù|ù',   r'{u`}',  txt)  # u-grave
+    txt = re.sub(u'ú|ú|ú',   r"{u'}",  txt)  # u-acute
+    txt = re.sub(u'û|û|û',    r'{u^}', txt)  # u-circumflex
+    txt = re.sub(u'ü|ü|ü',     r'{u"}',  txt)  # u-umlaut
+    txt = re.sub(u'ý|ý|ý',   r"{y'}",  txt)  # y-acute
+    txt = re.sub(u'ÿ|ÿ|ÿ',     r'{y"}',  txt)  # y-umlaut
+    txt = re.sub(u'Œ|Œ|Œ',    r'{OE}',  txt)  # OE
+    txt = re.sub(u'œ|œ|œ',    r'{oe}',  txt)  # oe
+    txt = re.sub(u'Ŝ|Š|Ŝ',   r'{S^}', txt)  # Scaron
+    txt = re.sub(u'ŝ|š|ŝ',   r'{s^}', txt)  # scaron
+    txt = re.sub(u'•|•|•',    r'{*}',   txt)  # bullet
+    txt = re.sub(u'₣|₣',           r'{Fr}',  txt)  # Franc
+    txt = re.sub(u'₤|₤',           r'{L=}',  txt)  # Lira
+    txt = re.sub(u'₨|₨',           r'{Rs}',  txt)  # Rupee
+    txt = re.sub(u'€|€|€',    r'{C=}',  txt)  # euro
+    txt = re.sub(u'™|™|™',   r'{tm}',  txt)  # trademark
+    txt = re.sub(u'♠|♠|♠',  r'{spade}',   txt)  # spade
+    txt = re.sub(u'♣|♣|♣',   r'{club}',    txt)  # club
+    txt = re.sub(u'♥|♥|♥',  r'{heart}',   txt)  # heart
+    txt = re.sub(u'♦|♦|♦',   r'{diamond}', txt)  # diamond
+
+    # Move into main code?
+#    txt = re.sub(u'\xa0',   r'p. ', txt)              # blank paragraph
+#    txt = re.sub(u'\n\n\n\n',   r'\n\np. \n\n', txt)  # blank paragraph
+#    txt = re.sub(u'\n  \n',   r'\n<br />\n', txt)     # blank paragraph - br tag
+
+    return txt

src/calibre/ebooks/unihandecode/pykakasi/jisyo.py

@@ -2,12 +2,8 @@
 #  jisyo.py
 #
 # Copyright 2011 Hiroshi Miura <miurahr@linux.com>
-from cPickle import load
-import anydbm,marshal
+import cPickle, marshal
 from zlib import decompress
-import os
-
-import calibre.utils.resources as resources
 
 class jisyo (object):
     kanwadict = None
@@ -25,16 +21,14 @@ class jisyo (object):
 
     def __init__(self):
         if self.kanwadict is None:
-            dictpath = resources.get_path(os.path.join('localization','pykakasi','kanwadict2.db'))
-            self.kanwadict = anydbm.open(dictpath,'r')
-        if self.itaijidict is  None:
-            itaijipath = resources.get_path(os.path.join('localization','pykakasi','itaijidict2.pickle'))
-            itaiji_pkl = open(itaijipath, 'rb')
-            self.itaijidict = load(itaiji_pkl)
+            self.kanwadict = cPickle.loads(
+                P('localization/pykakasi/kanwadict2.pickle', data=True))
+        if self.itaijidict is None:
+            self.itaijidict = cPickle.loads(
+                P('localization/pykakasi/itaijidict2.pickle', data=True))
         if self.kanadict is None:
-            kanadictpath = resources.get_path(os.path.join('localization','pykakasi','kanadict2.pickle'))
-            kanadict_pkl = open(kanadictpath, 'rb')
-            self.kanadict = load(kanadict_pkl)
+            self.kanadict = cPickle.loads(
+                P('localization/pykakasi/kanadict2.pickle', data=True))
 
     def load_jisyo(self, char):
         try:#python2

src/calibre/gui2/__init__.py

@@ -620,7 +620,22 @@ class Application(QApplication):
         self.original_font = QFont(QApplication.font())
         fi = gprefs['font']
         if fi is not None:
-            QApplication.setFont(QFont(*fi))
+            font = QFont(*(fi[:4]))
+            s = gprefs.get('font_stretch', None)
+            if s is not None:
+                font.setStretch(s)
+            QApplication.setFont(font)
+        st = self.style()
+        if st is not None:
+            st = unicode(st.objectName()).lower()
+        if (islinux or isfreebsd) and st in ('windows', 'motif', 'cde'):
+            from PyQt4.Qt import QStyleFactory
+            styles = set(map(unicode, QStyleFactory.keys()))
+            if 'Plastique' in styles and os.environ.get('KDE_FULL_SESSION',
+                    False):
+                self.setStyle('Plastique')
+            elif 'Cleanlooks' in styles:
+                self.setStyle('Cleanlooks')
 
     def _send_file_open_events(self):
         with self._file_open_lock:

src/calibre/gui2/actions/delete.py

@@ -9,11 +9,12 @@ from functools import partial
 
 from PyQt4.Qt import QMenu, QObject, QTimer
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.dialogs.delete_matching_from_device import DeleteMatchingFromDeviceDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.confirm_delete_location import confirm_location
 from calibre.gui2.actions import InterfaceAction
+from calibre.utils.recycle_bin import can_recycle
 
 single_shot = partial(QTimer.singleShot, 10)
 
@@ -24,6 +25,15 @@ class MultiDeleter(QObject):
         QObject.__init__(self, gui)
         self.model = gui.library_view.model()
         self.ids = ids
+        self.permanent = False
+        if can_recycle and len(ids) > 100:
+            if question_dialog(gui, _('Are you sure?'), '<p>'+
+                _('You are trying to delete %d books. '
+                    'Sending so many files to the Recycle'
+                    ' Bin <b>can be slow</b>. Should calibre skip the'
+                    ' Recycle Bin? If you click Yes the files'
+                    ' will be <b>permanently deleted</b>.')%len(ids)):
+                self.permanent = True
         self.gui = gui
         self.failures = []
         self.deleted_ids = []
@@ -44,7 +54,8 @@ class MultiDeleter(QObject):
             title_ = self.model.db.title(id_, index_is_id=True)
             if title_:
                 title = title_
-            self.model.db.delete_book(id_, notify=False, commit=False)
+            self.model.db.delete_book(id_, notify=False, commit=False,
+                    permanent=self.permanent)
             self.deleted_ids.append(id_)
         except:
             import traceback

src/calibre/gui2/actions/edit_metadata.py

@@ -478,6 +478,10 @@ class EditMetadataAction(InterfaceAction):
         try:
             set_title = not mi.is_null('title')
             set_authors = not mi.is_null('authors')
+            idents = db.get_identifiers(i, index_is_id=True)
+            if mi.identifiers:
+                idents.update(mi.identifiers)
+            mi.identifiers = idents
             db.set_metadata(i, mi, commit=False, set_title=set_title,
                     set_authors=set_authors, notify=False)
             self.applied_ids.append(i)

src/calibre/gui2/actions/preferences.py

@@ -19,8 +19,9 @@ class PreferencesAction(InterfaceAction):
 
     def genesis(self):
         pm = QMenu()
-        acname = _('Change calibre behavior') if isosx else _('Preferences')
-        pm.addAction(QIcon(I('config.png')), acname, self.do_config)
+        pm.addAction(QIcon(I('config.png')), _('Preferences'), self.do_config)
+        if isosx:
+            pm.addAction(QIcon(I('config.png')), _('Change calibre behavior'), self.do_config)
         pm.addAction(QIcon(I('wizard.png')), _('Run welcome wizard'),
                 self.gui.run_wizard)
         if not DEBUG:

src/calibre/gui2/actions/store.py

@@ -10,6 +10,7 @@ from functools import partial
 
 from PyQt4.Qt import QMenu
 
+from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.confirm_delete import confirm
 
@@ -19,24 +20,93 @@ class StoreAction(InterfaceAction):
     action_spec = (_('Get books'), 'store.png', None, None)
 
     def genesis(self):
-        self.qaction.triggered.connect(self.search)
+        self.qaction.triggered.connect(self.do_search)
         self.store_menu = QMenu()
         self.load_menu()
 
     def load_menu(self):
         self.store_menu.clear()
-        self.store_menu.addAction(_('Search'), self.search)
+        self.store_menu.addAction(_('Search for ebooks'), self.search)
+        self.store_menu.addAction(_('Search for this author'), self.search_author)
+        self.store_menu.addAction(_('Search for this title'), self.search_title)
+        self.store_menu.addAction(_('Search for this book'), self.search_author_title)
         self.store_menu.addSeparator()
-        for n, p in self.gui.istores.items():
-            self.store_menu.addAction(n, partial(self.open_store, p))
+        self.store_list_menu = self.store_menu.addMenu(_('Stores'))
+        for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()):
+            self.store_list_menu.addAction(n, partial(self.open_store, p))
         self.qaction.setMenu(self.store_menu)
 
-    def search(self):
+    def do_search(self):
+        return self.search()
+
+    def search(self, query=''):
         self.show_disclaimer()
         from calibre.gui2.store.search.search import SearchDialog
-        sd = SearchDialog(self.gui.istores, self.gui)
+        sd = SearchDialog(self.gui.istores, self.gui, query)
         sd.exec_()
 
+    def _get_selected_row(self):
+        rows = self.gui.current_view().selectionModel().selectedRows()
+        if not rows or len(rows) == 0:
+            return None
+        return rows[0].row()
+
+    def _get_author(self, row):
+        authors = []
+
+        if self.gui.current_view() is self.gui.library_view:
+            a = self.gui.library_view.model().authors(row)
+            authors = a.split(',')
+        else:
+            mi = self.gui.current_view().model().get_book_display_info(row)
+            authors = mi.authors
+
+        corrected_authors = []
+        for x in authors:
+            a = x.split('|')
+            a.reverse()
+            a = ' '.join(a)
+            corrected_authors.append(a)
+
+        return ' & '.join(corrected_authors).strip()
+
+    def search_author(self):
+        row = self._get_selected_row()
+        if row == None:
+            error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True)
+            return
+
+        query = 'author:"%s"' % self._get_author(row)
+        self.search(query)
+
+    def _get_title(self, row):
+        title = ''
+        if self.gui.current_view() is self.gui.library_view:
+            title = self.gui.library_view.model().title(row)
+        else:
+            mi = self.gui.current_view().model().get_book_display_info(row)
+            title = mi.title
+
+        return title.strip()
+
+    def search_title(self):
+        row = self._get_selected_row()
+        if row == None:
+            error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True)
+            return
+
+        query = 'title:"%s"' % self._get_title(row)
+        self.search(query)
+
+    def search_author_title(self):
+        row = self._get_selected_row()
+        if row == None:
+            error_dialog(self.gui, _('Cannot search'), _('No book selected'), show=True)
+            return
+
+        query = 'author:"%s" title:"%s"' % (self._get_author(row), self._get_title(row))
+        self.search(query)
+
     def open_store(self, store_plugin):
         self.show_disclaimer()
         store_plugin.open(self.gui)

src/calibre/gui2/bars.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import (QObject, QToolBar, Qt, QSize, QToolButton, QVBoxLayout,
+        QLabel, QWidget, QAction, QMenuBar, QMenu)
+
+from calibre.constants import isosx
+from calibre.gui2 import gprefs
+
+class ToolBar(QToolBar): # {{{
+
+    def __init__(self, donate, location_manager, parent):
+        QToolBar.__init__(self, parent)
+        self.setContextMenuPolicy(Qt.PreventContextMenu)
+        self.setMovable(False)
+        self.setFloatable(False)
+        self.setOrientation(Qt.Horizontal)
+        self.setAllowedAreas(Qt.TopToolBarArea|Qt.BottomToolBarArea)
+        self.setStyleSheet('QToolButton:checked { font-weight: bold }')
+        self.preferred_width = self.sizeHint().width()
+        self.gui = parent
+        self.donate_button = donate
+        self.added_actions = []
+
+        self.location_manager = location_manager
+        donate.setAutoRaise(True)
+        donate.setCursor(Qt.PointingHandCursor)
+        self.setAcceptDrops(True)
+        self.showing_donate = False
+
+    def resizeEvent(self, ev):
+        QToolBar.resizeEvent(self, ev)
+        style = self.get_text_style()
+        self.setToolButtonStyle(style)
+        if hasattr(self, 'd_widget') and hasattr(self.d_widget, 'filler'):
+            self.d_widget.filler.setVisible(style != Qt.ToolButtonIconOnly)
+
+    def get_text_style(self):
+        style = Qt.ToolButtonTextUnderIcon
+        s = gprefs['toolbar_icon_size']
+        if s != 'off':
+            p = gprefs['toolbar_text']
+            if p == 'never':
+                style = Qt.ToolButtonIconOnly
+            elif p == 'auto' and self.preferred_width > self.width()+35:
+                style = Qt.ToolButtonIconOnly
+        return style
+
+    def contextMenuEvent(self, *args):
+        pass
+
+    def update_lm_actions(self):
+        for ac in self.added_actions:
+            if ac in self.location_manager.all_actions:
+                ac.setVisible(ac in self.location_manager.available_actions)
+
+    def init_bar(self, actions):
+        self.showing_donate = False
+        for ac in self.added_actions:
+            m = ac.menu()
+            if m is not None:
+                m.setVisible(False)
+
+        self.clear()
+        self.added_actions = []
+
+        bar = self
+
+        for what in actions:
+            if what is None:
+                bar.addSeparator()
+            elif what == 'Location Manager':
+                for ac in self.location_manager.all_actions:
+                    bar.addAction(ac)
+                    bar.added_actions.append(ac)
+                    bar.setup_tool_button(bar, ac, QToolButton.MenuButtonPopup)
+                    ac.setVisible(False)
+            elif what == 'Donate':
+                self.d_widget = QWidget()
+                self.d_widget.setLayout(QVBoxLayout())
+                self.d_widget.layout().addWidget(self.donate_button)
+                if isosx:
+                    self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
+                    self.d_widget.layout().setContentsMargins(0,0,0,0)
+                    self.d_widget.setContentsMargins(0,0,0,0)
+                    self.d_widget.filler = QLabel(u'\u00a0')
+                    self.d_widget.layout().addWidget(self.d_widget.filler)
+                bar.addWidget(self.d_widget)
+                self.showing_donate = True
+            elif what in self.gui.iactions:
+                action = self.gui.iactions[what]
+                bar.addAction(action.qaction)
+                self.added_actions.append(action.qaction)
+                self.setup_tool_button(bar, action.qaction, action.popup_type)
+        self.preferred_width = self.sizeHint().width()
+
+    def setup_tool_button(self, bar, ac, menu_mode=None):
+        ch = bar.widgetForAction(ac)
+        if ch is None:
+            ch = self.child_bar.widgetForAction(ac)
+        ch.setCursor(Qt.PointingHandCursor)
+        ch.setAutoRaise(True)
+        if ac.menu() is not None and menu_mode is not None:
+            ch.setPopupMode(menu_mode)
+        return ch
+
+    #support drag&drop from/to library from/to reader/card
+    def dragEnterEvent(self, event):
+        md = event.mimeData()
+        if md.hasFormat("application/calibre+from_library") or \
+           md.hasFormat("application/calibre+from_device"):
+            event.setDropAction(Qt.CopyAction)
+            event.accept()
+        else:
+            event.ignore()
+
+    def dragMoveEvent(self, event):
+        allowed = False
+        md = event.mimeData()
+        #Drop is only allowed in the location manager widget's different from the selected one
+        for ac in self.location_manager.available_actions:
+            w = self.widgetForAction(ac)
+            if w is not None:
+                if ( md.hasFormat("application/calibre+from_library") or \
+                     md.hasFormat("application/calibre+from_device") ) and \
+                        w.geometry().contains(event.pos()) and \
+                        isinstance(w, QToolButton) and not w.isChecked():
+                    allowed = True
+                    break
+        if allowed:
+            event.acceptProposedAction()
+        else:
+            event.ignore()
+
+    def dropEvent(self, event):
+        data = event.mimeData()
+
+        mime = 'application/calibre+from_library'
+        if data.hasFormat(mime):
+            ids = list(map(int, str(data.data(mime)).split()))
+            tgt = None
+            for ac in self.location_manager.available_actions:
+                w = self.widgetForAction(ac)
+                if w is not None and w.geometry().contains(event.pos()):
+                    tgt = ac.calibre_name
+            if tgt is not None:
+                if tgt == 'main':
+                    tgt = None
+                self.gui.sync_to_device(tgt, False, send_ids=ids)
+                event.accept()
+
+        mime = 'application/calibre+from_device'
+        if data.hasFormat(mime):
+            paths = [unicode(u.toLocalFile()) for u in data.urls()]
+            if paths:
+                self.gui.iactions['Add Books'].add_books_from_device(
+                        self.gui.current_view(), paths=paths)
+                event.accept()
+
+# }}}
+
+class MenuAction(QAction): # {{{
+
+    def __init__(self, clone, parent):
+        QAction.__init__(self, clone.text(), parent)
+        self.clone = clone
+        clone.changed.connect(self.clone_changed)
+
+    def clone_changed(self):
+        self.setText(self.clone.text())
+# }}}
+
+class MenuBar(QMenuBar): # {{{
+
+    def __init__(self, location_manager, parent):
+        QMenuBar.__init__(self, parent)
+        self.gui = parent
+        self.setNativeMenuBar(True)
+
+        self.location_manager = location_manager
+        self.added_actions = []
+
+        self.donate_action = QAction(_('Donate'), self)
+        self.donate_menu = QMenu()
+        self.donate_menu.addAction(self.gui.donate_action)
+        self.donate_action.setMenu(self.donate_menu)
+
+    def update_lm_actions(self):
+        for ac in self.added_actions:
+            if ac in self.location_manager.all_actions:
+                ac.setVisible(ac in self.location_manager.available_actions)
+
+    def init_bar(self, actions):
+        for ac in self.added_actions:
+            m = ac.menu()
+            if m is not None:
+                m.setVisible(False)
+
+        self.clear()
+        self.added_actions = []
+
+        for what in actions:
+            if what is None:
+                continue
+            elif what == 'Location Manager':
+                for ac in self.location_manager.all_actions:
+                    ac = self.build_menu(ac)
+                    self.addAction(ac)
+                    self.added_actions.append(ac)
+                    ac.setVisible(False)
+            elif what == 'Donate':
+                self.addAction(self.donate_action)
+            elif what in self.gui.iactions:
+                action = self.gui.iactions[what]
+                ac = self.build_menu(action.qaction)
+                self.addAction(ac)
+                self.added_actions.append(ac)
+
+    def build_menu(self, action):
+        m = action.menu()
+        ac = MenuAction(action, self)
+        if m is None:
+            m = QMenu()
+            m.addAction(action)
+        ac.setMenu(m)
+        return ac
+
+# }}}
+
+class BarsManager(QObject):
+
+    def __init__(self, donate_button, location_manager, parent):
+        QObject.__init__(self, parent)
+        self.donate_button, self.location_manager = (donate_button,
+                location_manager)
+
+        bars = [ToolBar(donate_button, location_manager, parent) for i in
+                range(3)]
+        self.main_bars = tuple(bars[:2])
+        self.child_bars = tuple(bars[2:])
+
+        self.menu_bar = MenuBar(self.location_manager, self.parent())
+        self.parent().setMenuBar(self.menu_bar)
+
+        self.apply_settings()
+        self.init_bars()
+
+    def database_changed(self, db):
+        pass
+
+    @property
+    def bars(self):
+        for x in self.main_bars + self.child_bars:
+            yield x
+
+    @property
+    def showing_donate(self):
+        for b in self.bars:
+            if b.isVisible() and b.showing_donate:
+                return True
+        return False
+
+    def init_bars(self):
+        self.bar_actions = tuple(
+            [gprefs['action-layout-toolbar'+x] for x in ('', '-device')] +
+            [gprefs['action-layout-toolbar-child']] +
+            [gprefs['action-layout-menubar']] +
+            [gprefs['action-layout-menubar-device']]
+        )
+
+        for bar, actions in zip(self.bars, self.bar_actions[:3]):
+            bar.init_bar(actions)
+
+    def update_bars(self):
+        '''
+        This shows the correct main toolbar and rebuilds the menubar based on
+        whether a device is connected or not. Note that the toolbars are
+        explicitly not rebuilt, this is to workaround a Qt limitation iwth
+        QToolButton's popup menus and modal dialogs. If you want the toolbars
+        rebuilt, call init_bars().
+        '''
+        showing_device = self.location_manager.has_device
+        main_bar = self.main_bars[1 if showing_device else 0]
+        child_bar = self.child_bars[0]
+        for bar in self.bars:
+            bar.setVisible(False)
+            bar.update_lm_actions()
+        if main_bar.added_actions:
+            main_bar.setVisible(True)
+        if child_bar.added_actions:
+            child_bar.setVisible(True)
+
+        self.menu_bar.init_bar(self.bar_actions[4 if showing_device else 3])
+        self.menu_bar.update_lm_actions()
+        self.menu_bar.setVisible(bool(self.menu_bar.added_actions))
+
+    def apply_settings(self):
+        sz = gprefs['toolbar_icon_size']
+        sz = {'off':0, 'small':24, 'medium':48, 'large':64}[sz]
+        style = Qt.ToolButtonTextUnderIcon
+        if sz > 0 and gprefs['toolbar_text'] == 'never':
+            style = Qt.ToolButtonIconOnly
+
+        for bar in self.bars:
+            bar.setIconSize(QSize(sz, sz))
+            bar.setToolButtonStyle(style)
+        self.donate_button.set_normal_icon_size(sz, sz)
+
+

src/calibre/gui2/convert/txt_output.py

@@ -19,7 +19,7 @@ class PluginWidget(Widget, Ui_Form):
         Widget.__init__(self, parent,
         ['newline', 'max_line_length', 'force_max_line_length',
         'inline_toc', 'txt_output_formatting', 'keep_links', 'keep_image_references',
-        'txt_output_encoding'])
+        'keep_color', 'txt_output_encoding'])
         self.db, self.book_id = db, book_id
         for x in get_option('newline').option.choices:
             self.opt_newline.addItem(x)

src/calibre/gui2/convert/txt_output.ui

@@ -122,6 +122,13 @@
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QCheckBox" name="opt_keep_color">
+        <property name="text">
+         <string>Keep text color, when possible</string>
+        </property>
+       </widget>
+      </item>
      </layout>
     </widget>
    </item>

src/calibre/gui2/device.py

@@ -751,6 +751,7 @@ class DeviceMixin(object): # {{{
             if self.current_view() != self.library_view:
                 self.book_details.reset_info()
             self.location_manager.update_devices()
+            self.bars_manager.update_bars()
             self.library_view.set_device_connected(self.device_connected)
             self.refresh_ondevice()
         device_signals.device_connection_changed.emit(connected)
@@ -764,6 +765,7 @@ class DeviceMixin(object): # {{{
         info, cp, fs = job.result
         self.location_manager.update_devices(cp, fs,
                 self.device_manager.device.icon)
+        self.bars_manager.update_bars()
         self.status_bar.device_connected(info[0])
         self.device_manager.books(Dispatcher(self.metadata_downloaded))
 

src/calibre/gui2/dialogs/edit_authors_dialog.ui

@@ -78,22 +78,22 @@
     </widget>
    </item>
    <item>
-    <layout class="QHBoxLayout" name="horizontalLayout">
-     <item>
+    <layout class="QGridLayout">
+     <item row="0" column="0">
       <widget class="QPushButton" name="sort_by_author">
        <property name="text">
         <string>Sort by author</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="0" column="1">
       <widget class="QPushButton" name="sort_by_author_sort">
        <property name="text">
         <string>Sort by author sort</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="1" column="0">
       <widget class="QPushButton" name="recalc_author_sort">
        <property name="toolTip">
         <string>Reset all the author sort values to a value automatically
@@ -105,7 +105,7 @@ generated can be controlled via Preferences-&gt;Advanced-&gt;Tweaks</string>
        </property>
       </widget>
      </item>
-     <item>
+     <item row="1" column="1">
       <widget class="QPushButton" name="auth_sort_to_author">
        <property name="toolTip">
         <string>Copy author sort to author for every author. You typically use this button
@@ -116,20 +116,7 @@ after changing Preferences-&gt;Advanced-&gt;Tweaks-&gt;Author sort name algorith
        </property>
       </widget>
      </item>
-     <item>
-      <spacer name="horizontalSpacer_3">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
+     <item row="1" column="2">
       <widget class="QDialogButtonBox" name="buttonBox">
        <property name="sizePolicy">
         <sizepolicy hsizetype="Minimum" vsizetype="Fixed">

src/calibre/gui2/layout.py

@@ -7,16 +7,17 @@ __docformat__ = 'restructuredtext en'
 
 from functools import partial
 
-from PyQt4.Qt import (QIcon, Qt, QWidget, QToolBar, QSize,
-    pyqtSignal, QToolButton, QMenu, QMenuBar, QAction,
+from PyQt4.Qt import (QIcon, Qt, QWidget, QSize,
+    pyqtSignal, QToolButton, QMenu,
     QObject, QVBoxLayout, QSizePolicy, QLabel, QHBoxLayout, QActionGroup)
 
 
-from calibre.constants import __appname__, isosx
+from calibre.constants import __appname__
 from calibre.gui2.search_box import SearchBox2, SavedSearchBox
 from calibre.gui2.throbber import ThrobbingButton
-from calibre.gui2 import gprefs
+from calibre.gui2.bars import BarsManager
 from calibre.gui2.widgets import ComboBoxWithHelp
+from calibre.utils.config_base import tweaks
 from calibre import human_readable
 
 class LocationManager(QObject): # {{{
@@ -35,6 +36,8 @@ class LocationManager(QObject): # {{{
         self._mem = []
         self.tooltips = {}
 
+        self.all_actions = []
+
         def ac(name, text, icon, tooltip):
             icon = QIcon(I(icon))
             ac = self.location_actions.addAction(icon, text)
@@ -59,6 +62,7 @@ class LocationManager(QObject): # {{{
             ac.setMenu(m)
             ac.calibre_name = name
 
+            self.all_actions.append(ac)
             return ac
 
         self.library_action = ac('library', _('Library'), 'lt.png',
@@ -234,259 +238,6 @@ class Spacer(QWidget): # {{{
         self.l.addStretch(10)
 # }}}
 
-class MenuAction(QAction): # {{{
-
-    def __init__(self, clone, parent):
-        QAction.__init__(self, clone.text(), parent)
-        self.clone = clone
-        clone.changed.connect(self.clone_changed)
-
-    def clone_changed(self):
-        self.setText(self.clone.text())
-# }}}
-
-class MenuBar(QMenuBar): # {{{
-
-    def __init__(self, location_manager, parent):
-        QMenuBar.__init__(self, parent)
-        self.gui = parent
-        self.setNativeMenuBar(True)
-
-        self.location_manager = location_manager
-        self.location_manager.locations_changed.connect(self.build_bar)
-        self.added_actions = []
-
-        self.donate_action = QAction(_('Donate'), self)
-        self.donate_menu = QMenu()
-        self.donate_menu.addAction(self.gui.donate_action)
-        self.donate_action.setMenu(self.donate_menu)
-        self.build_bar()
-
-    def build_bar(self, changed_action=None):
-        showing_device = self.location_manager.has_device
-        actions = '-device' if showing_device else ''
-        actions = gprefs['action-layout-menubar'+actions]
-
-        show_main = len(actions) > 0
-        self.setVisible(show_main)
-
-        for ac in self.added_actions:
-            m = ac.menu()
-            if m is not None:
-                m.setVisible(False)
-
-        self.clear()
-        self.added_actions = []
-        self.action_map = {}
-
-        for what in actions:
-            if what is None:
-                continue
-            elif what == 'Location Manager':
-                for ac in self.location_manager.available_actions:
-                    ac = self.build_menu(ac)
-                    self.addAction(ac)
-                    self.added_actions.append(ac)
-            elif what == 'Donate':
-                self.addAction(self.donate_action)
-            elif what in self.gui.iactions:
-                action = self.gui.iactions[what]
-                ac = self.build_menu(action.qaction)
-                self.addAction(ac)
-                self.added_actions.append(ac)
-
-    def build_menu(self, action):
-        m = action.menu()
-        ac = MenuAction(action, self)
-        if m is None:
-            m = QMenu()
-            m.addAction(action)
-        ac.setMenu(m)
-        return ac
-
-# }}}
-
-class BaseToolBar(QToolBar): # {{{
-
-    def __init__(self, parent):
-        QToolBar.__init__(self, parent)
-        self.setContextMenuPolicy(Qt.PreventContextMenu)
-        self.setMovable(False)
-        self.setFloatable(False)
-        self.setOrientation(Qt.Horizontal)
-        self.setAllowedAreas(Qt.TopToolBarArea|Qt.BottomToolBarArea)
-        self.setStyleSheet('QToolButton:checked { font-weight: bold }')
-        self.preferred_width = self.sizeHint().width()
-
-    def resizeEvent(self, ev):
-        QToolBar.resizeEvent(self, ev)
-        style = self.get_text_style()
-        self.setToolButtonStyle(style)
-        if hasattr(self, 'd_widget') and hasattr(self.d_widget, 'filler'):
-            self.d_widget.filler.setVisible(style != Qt.ToolButtonIconOnly)
-
-    def get_text_style(self):
-        style = Qt.ToolButtonTextUnderIcon
-        s = gprefs['toolbar_icon_size']
-        if s != 'off':
-            p = gprefs['toolbar_text']
-            if p == 'never':
-                style = Qt.ToolButtonIconOnly
-            elif p == 'auto' and self.preferred_width > self.width()+35:
-                style = Qt.ToolButtonIconOnly
-        return style
-
-    def contextMenuEvent(self, *args):
-        pass
-
-# }}}
-
-class ToolBar(BaseToolBar): # {{{
-
-    def __init__(self, donate, location_manager, child_bar, parent):
-        BaseToolBar.__init__(self, parent)
-        self.gui = parent
-        self.child_bar = child_bar
-        self.donate_button = donate
-        self.apply_settings()
-
-        self.location_manager = location_manager
-        self.location_manager.locations_changed.connect(self.build_bar)
-        donate.setAutoRaise(True)
-        donate.setCursor(Qt.PointingHandCursor)
-        self.added_actions = []
-        self.build_bar()
-        self.setAcceptDrops(True)
-
-    def apply_settings(self):
-        sz = gprefs['toolbar_icon_size']
-        sz = {'off':0, 'small':24, 'medium':48, 'large':64}[sz]
-        self.setIconSize(QSize(sz, sz))
-        self.child_bar.setIconSize(QSize(sz, sz))
-        style = Qt.ToolButtonTextUnderIcon
-        if sz > 0 and gprefs['toolbar_text'] == 'never':
-            style = Qt.ToolButtonIconOnly
-        self.setToolButtonStyle(style)
-        self.child_bar.setToolButtonStyle(style)
-        self.donate_button.set_normal_icon_size(sz, sz)
-
-    def build_bar(self):
-        self.showing_donate = False
-        showing_device = self.location_manager.has_device
-        mactions = '-device' if showing_device else ''
-        mactions = gprefs['action-layout-toolbar'+mactions]
-        cactions = gprefs['action-layout-toolbar-child']
-
-        show_main = len(mactions) > 0
-        self.setVisible(show_main)
-        show_child = len(cactions) > 0
-        self.child_bar.setVisible(show_child)
-
-        for ac in self.added_actions:
-            m = ac.menu()
-            if m is not None:
-                m.setVisible(False)
-
-        self.clear()
-        self.child_bar.clear()
-        self.added_actions = []
-
-        for bar, actions in ((self, mactions), (self.child_bar, cactions)):
-            for what in actions:
-                if what is None:
-                    bar.addSeparator()
-                elif what == 'Location Manager':
-                    for ac in self.location_manager.available_actions:
-                        bar.addAction(ac)
-                        bar.added_actions.append(ac)
-                        bar.setup_tool_button(bar, ac, QToolButton.MenuButtonPopup)
-                elif what == 'Donate':
-                    self.d_widget = QWidget()
-                    self.d_widget.setLayout(QVBoxLayout())
-                    self.d_widget.layout().addWidget(self.donate_button)
-                    if isosx:
-                        self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }')
-                        self.d_widget.layout().setContentsMargins(0,0,0,0)
-                        self.d_widget.setContentsMargins(0,0,0,0)
-                        self.d_widget.filler = QLabel(u'\u00a0')
-                        self.d_widget.layout().addWidget(self.d_widget.filler)
-                    bar.addWidget(self.d_widget)
-                    self.showing_donate = True
-                elif what in self.gui.iactions:
-                    action = self.gui.iactions[what]
-                    bar.addAction(action.qaction)
-                    self.added_actions.append(action.qaction)
-                    self.setup_tool_button(bar, action.qaction, action.popup_type)
-        self.preferred_width = self.sizeHint().width()
-        self.child_bar.preferred_width = self.child_bar.sizeHint().width()
-
-    def setup_tool_button(self, bar, ac, menu_mode=None):
-        ch = bar.widgetForAction(ac)
-        if ch is None:
-            ch = self.child_bar.widgetForAction(ac)
-        ch.setCursor(Qt.PointingHandCursor)
-        ch.setAutoRaise(True)
-        if ac.menu() is not None and menu_mode is not None:
-            ch.setPopupMode(menu_mode)
-        return ch
-
-    def database_changed(self, db):
-        pass
-
-    #support drag&drop from/to library from/to reader/card
-    def dragEnterEvent(self, event):
-        md = event.mimeData()
-        if md.hasFormat("application/calibre+from_library") or \
-           md.hasFormat("application/calibre+from_device"):
-            event.setDropAction(Qt.CopyAction)
-            event.accept()
-        else:
-            event.ignore()
-
-    def dragMoveEvent(self, event):
-        allowed = False
-        md = event.mimeData()
-        #Drop is only allowed in the location manager widget's different from the selected one
-        for ac in self.location_manager.available_actions:
-            w = self.widgetForAction(ac)
-            if w is not None:
-                if ( md.hasFormat("application/calibre+from_library") or \
-                     md.hasFormat("application/calibre+from_device") ) and \
-                        w.geometry().contains(event.pos()) and \
-                        isinstance(w, QToolButton) and not w.isChecked():
-                    allowed = True
-                    break
-        if allowed:
-            event.acceptProposedAction()
-        else:
-            event.ignore()
-
-    def dropEvent(self, event):
-        data = event.mimeData()
-
-        mime = 'application/calibre+from_library'
-        if data.hasFormat(mime):
-            ids = list(map(int, str(data.data(mime)).split()))
-            tgt = None
-            for ac in self.location_manager.available_actions:
-                w = self.widgetForAction(ac)
-                if w is not None and w.geometry().contains(event.pos()):
-                    tgt = ac.calibre_name
-            if tgt is not None:
-                if tgt == 'main':
-                    tgt = None
-                self.gui.sync_to_device(tgt, False, send_ids=ids)
-                event.accept()
-
-        mime = 'application/calibre+from_device'
-        if data.hasFormat(mime):
-            paths = [unicode(u.toLocalFile()) for u in data.urls()]
-            if paths:
-                self.gui.iactions['Add Books'].add_books_from_device(
-                        self.gui.current_view(), paths=paths)
-                event.accept()
-
-# }}}
 
 class MainWindowMixin(object): # {{{
 
@@ -507,14 +258,17 @@ class MainWindowMixin(object): # {{{
         self.iactions['Fetch News'].init_scheduler(db)
 
         self.search_bar = SearchBar(self)
-        self.child_bar = BaseToolBar(self)
-        self.tool_bar = ToolBar(self.donate_button,
-                self.location_manager, self.child_bar, self)
-        self.addToolBar(Qt.TopToolBarArea, self.tool_bar)
-        self.addToolBar(Qt.BottomToolBarArea, self.child_bar)
-        self.menu_bar = MenuBar(self.location_manager, self)
-        self.setMenuBar(self.menu_bar)
-        self.setUnifiedTitleAndToolBarOnMac(True)
+        self.bars_manager = BarsManager(self.donate_button,
+                self.location_manager, self)
+        for bar in self.bars_manager.main_bars:
+            self.addToolBar(Qt.TopToolBarArea, bar)
+        for bar in self.bars_manager.child_bars:
+            self.addToolBar(Qt.BottomToolBarArea, bar)
+        self.bars_manager.update_bars()
+        # This is disabled because it introduces various toolbar related bugs
+        # The width of the toolbar becomes the sum of both toolbars
+        if tweaks['unified_title_toolbar_on_osx']:
+            self.setUnifiedTitleAndToolBarOnMac(True)
 
         l = self.centralwidget.layout()
         l.addWidget(self.search_bar)