Merge from trunk

Changelog.yaml

@@ -19,6 +19,60 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.7.57
+  date: 2011-04-22
+
+  new features:
+    - title: "Launch worker processes on demand instead of keeping a pool of them in memory. Reduces memory footprint."
+
+    - title: "Use the visual formatting of the Table of Contents to try to automatically create a multi-level TOC when converting/viewing MOBI files."
+      tickets: [763681]
+
+    - title: "Add a new function booksize() to the template language to get the value of the size column in calibre."
+
+    - title: "Add support for using metadata plugboards with the content server (only with the epub format)"
+
+    - title: "Change default algorithm for automatically computing author sort to be more intelligent and handle the case when the author name has a comma in it"
+
+    - title: "Show cover size in the tooltips of the book details panel and book details popup window"
+  
+  bug fixes:
+    - title: "Dragging and dropping a cover onto the book details panel did not change the cover size"
+      tickets: [768332]
+
+    - title: "Fix non-escaped '|' when searching for commas in authors using REGEXP_MATCH"
+
+    - title: "Fix ratings in templates being multiplied by 2"
+
+    - title: "Fix adding a comma to custom series values when using completion."
+      tickets: [763788]
+
+    - title: "CHM Input: Another workaround for a Microsoft mess."
+      tickets: [763336]
+
+    - title: "Fix job count in the spinner not always being updated when a job completes"
+
+    - title: "Changing case only of a title does not update title sort"
+      tickets: [768904]
+
+  improved recipes:
+    - ecuisine.ro, egirl.ro and tabu.ro
+    - Daily Telegraph
+    - Handelsblatt
+    - Il Sole 24 Ore
+    - Newsweek
+    - Arcamax
+
+  new recipes:
+    - title: BabyOnline.ro
+      author: Silviu Cotoara
+
+    - title: "The Journal.ie"
+      author: Phil Burns
+
+    - title: "Der Spiegel"
+      author: Nikolas Mangold
+
 - version: 0.7.56
   date: 2011-04-17
 

recipes/babyonline.recipe

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+babyonline.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BabyOnline(BasicNewsRecipe):
+    title                 = u'Baby Online'
+    __author__            = u'Silviu Cotoar\u0103'
+    description           = u'De la p\u0103rinte la p\u0103rinte'
+    publisher             = u'Baby Online'
+    oldest_article        = 50
+    language              = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    category              = 'Ziare,Reviste,Copii,Mame'
+    encoding              = 'utf-8'
+    cover_url             = 'http://www.babyonline.ro/images/default/logo.gif'
+
+    conversion_options = {
+                                 'comments'   : description
+                                ,'tags'       : category
+                                ,'language'   : language
+                                ,'publisher'  : publisher
+                         }
+
+    keep_only_tags = [
+                        dict(name='div', attrs={'id':'article_container'})
+                     ]
+
+    remove_tags = [
+						dict(name='div', attrs={'id':'bar_nav'}),
+						dict(name='div', attrs={'id':'service_send'}),
+						dict(name='div', attrs={'id':'other_videos'}),
+						dict(name='div', attrs={'class':'dot_line_yellow'}),
+						dict(name='a', attrs={'class':'print'}),
+						dict(name='a', attrs={'class':'email'}),
+						dict(name='a', attrs={'class':'YM'}),
+						dict(name='a', attrs={'class':'comment'}),
+						dict(name='div', attrs={'class':'tombstone_cross'}),
+						dict(name='span', attrs={'class':'liketext'})
+                  ]
+
+    remove_tags_after = [
+						dict(name='div', attrs={'id':'service_send'})
+                        ]
+
+    feeds          = [
+                        (u'Feeds', u'http://www.babyonline.ro/rss_homepage.xml')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/daily_telegraph.recipe

@@ -61,6 +61,12 @@ class DailyTelegraph(BasicNewsRecipe):
                       (u'Entertainment News', u'http://feeds.news.com.au/public/rss/2.0/dtele_entertainment_news_201.xml'),
                       (u'Lifestyle News', u'http://feeds.news.com.au/public/rss/2.0/dtele_lifestyle_227.xml'),
                       (u'Music', u'http://feeds.news.com.au/public/rss/2.0/dtele_music_441.xml'),
+                      (u'Sport',
+                          u'http://feeds.news.com.au/public/rss/2.0/dtele_sport_203.xml'),
+                      (u'Soccer',
+                          u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_soccer_344.xml'),
+                      (u'Rugby Union',
+                          u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_rugby_union_342.xml'),
                       (u'Property Confidential', u'http://feeds.news.com.au/public/rss/2.0/dtele_property_confidential_463.xml'),
                       (u'Property - Your Space', u'http://feeds.news.com.au/public/rss/2.0/dtele_property_yourspace_462.xml'),
                       (u'Confidential News', u'http://feeds.news.com.au/public/rss/2.0/dtele_entertainment_confidential_252.xml'),

recipes/der_spiegel.recipe

@@ -0,0 +1,83 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Nikolas Mangold <nmangold at gmail.com>'
+'''
+spiegel.de
+'''
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre import strftime
+from calibre import re
+
+class DerSpiegel(BasicNewsRecipe):
+    title                  = 'Der Spiegel'
+    __author__             = 'Nikolas Mangold'
+    description            = 'Der Spiegel, Printed Edition. Access to paid content.'
+    publisher              = 'SPIEGEL-VERLAG RUDOLF AUGSTEIN GMBH & CO. KG'
+    category               = 'news, politics, Germany'
+    no_stylesheets         = True
+    encoding               = 'cp1252'
+    needs_subscription     = True
+    remove_empty_feeds     = True
+    delay                  = 1
+    PREFIX                 = 'http://m.spiegel.de'
+    INDEX                  = PREFIX + '/spiegel/print/epaper/index-heftaktuell.html'
+    use_embedded_content   = False
+    masthead_url = 'http://upload.wikimedia.org/wikipedia/en/thumb/1/17/Der_Spiegel_logo.svg/200px-Der_Spiegel_logo.svg.png'
+    language               = 'de'
+    publication_type       = 'magazine'
+    extra_css              = ' body{font-family: Arial,Helvetica,sans-serif} '
+    timefmt = '[%W/%Y]'
+    empty_articles = ['Titelbild']
+    preprocess_regexps = [
+        (re.compile(r'<p>&#9670;</p>', re.DOTALL|re.IGNORECASE), lambda match: '<hr>'),
+        ]
+
+    def get_browser(self):
+        def has_login_name(form):
+            try:
+                form.find_control(name="f.loginName")
+            except:
+                return False
+            else:
+                return True
+
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.PREFIX + '/meinspiegel/login.html')
+            br.select_form(predicate=has_login_name)
+            br['f.loginName'    ] = self.username
+            br['f.password'] = self.password
+            br.submit()
+        return br
+
+    remove_tags_before =  dict(attrs={'class':'spArticleContent'})
+    remove_tags_after  =  dict(attrs={'class':'spArticleCredit'})
+
+    def parse_index(self):
+        soup = self.index_to_soup(self.INDEX)
+
+        cover = soup.find('img', width=248)
+        if cover is not None:
+            self.cover_url = cover['src']
+
+        index = soup.find('dl')
+
+        feeds = []
+        for section in index.findAll('dt'):
+            section_title = self.tag_to_string(section).strip()
+            self.log('Found section ', section_title)
+
+            articles = []
+            for article in section.findNextSiblings(['dd','dt']):
+                if article.name == 'dt':
+                    break
+                link = article.find('a')
+                title = self.tag_to_string(link).strip()
+                if title in self.empty_articles:
+                    continue
+                self.log('Found article ', title)
+                url = self.PREFIX + link['href']
+                articles.append({'title' : title, 'date' : strftime(self.timefmt), 'url' : url})
+            feeds.append((section_title,articles))
+        return feeds;

recipes/ecuisine.recipe

@@ -14,14 +14,14 @@ class EcuisineRo(BasicNewsRecipe):
     __author__            = u'Silviu Cotoar\u0103'
     description           = u'Reinventeaz\u0103 pl\u0103cerea de a g\u0103ti'
     publisher             = 'eCuisine'
-    oldest_article        = 5
+    oldest_article        = 50
     language              = 'ro'
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     category              = 'Ziare,Retete,Bucatarie'
     encoding              = 'utf-8'
-    cover_url             = ''
+    cover_url             = 'http://www.ecuisine.ro/sites/all/themes/ecuisine/images/logo.gif'
 
     conversion_options = {
                              'comments'   : description
@@ -31,8 +31,8 @@ class EcuisineRo(BasicNewsRecipe):
                          }
 
     keep_only_tags = [
-            dict(name='div', attrs={'class':'page-title'})
+            dict(name='h1', attrs={'id':'page-title'})
-            , dict(name='div', attrs={'class':'content clearfix'})
+            , dict(name='div', attrs={'class':'field-item even'})
                      ]
 
     remove_tags = [

recipes/egirl.recipe

@@ -31,8 +31,8 @@ class EgirlRo(BasicNewsRecipe):
                          }
 
     keep_only_tags = [
-                          dict(name='div', attrs={'id':'title_art'})
+                          dict(name='div', attrs={'id':'content_art'})
-                        , dict(name='div', attrs={'class':'content_style'})
+                        , dict(name='div', attrs={'class':'content_articol'})
                      ]
 
     feeds          = [

recipes/handelsblatt.recipe

@@ -1,4 +1,3 @@
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Handelsblatt(BasicNewsRecipe):
@@ -7,14 +6,11 @@ class Handelsblatt(BasicNewsRecipe):
     oldest_article = 7
     max_articles_per_feed = 100
     no_stylesheets = True
-    cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png'
+#    cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png'
     language = 'de'
-  #  keep_only_tags = []
+
-    keep_only_tags = (dict(name = 'div', attrs = {'class': ['hcf-detail-abstract hcf-teaser ajaxify','hcf-detail','hcf-author-wrapper']}))
+    remove_tags_before =  dict(attrs={'class':'hcf-overline'})
-   # keep_only_tags.append(dict(name = 'div', attrs = {'id': 'fullText'}))
+    remove_tags_after  =  dict(attrs={'class':'hcf-footer'})
-    remove_tags    = [dict(name='img', attrs = {'src': 'http://www.handelsblatt.com/images/icon/loading.gif'})
-                      ,dict(name='ul' , attrs={'class':['hcf-detail-tools']})
-    									]
 
     feeds          = [
                         (u'Handelsblatt Exklusiv',u'http://www.handelsblatt.com/rss/exklusiv'),
@@ -28,17 +24,16 @@ class Handelsblatt(BasicNewsRecipe):
                         (u'Handelsblatt Magazin',u'http://www.handelsblatt.com/rss/magazin/'),
                         (u'Handelsblatt Weblogs',u'http://www.handelsblatt.com/rss/blogs')
                      ]
+
     extra_css = '''
-        .hcf-headline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;}
+        h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
-        .hcf-overline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;}
+        h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
-        .hcf-exclusive {font-family:Arial,Helvetica,sans-serif; font-style:italic;font-weight:bold; margin-right:5pt;}
+        p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
-        p{font-family:Arial,Helvetica,sans-serif;}
+        body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
-        .hcf-location-mark{font-weight:bold; margin-right:5pt;}
-        .MsoNormal{font-family:Helvetica,Arial,sans-serif;}
-        .hcf-author-wrapper{font-style:italic;}
-        .hcf-article-date{font-size:x-small;}
-        .hcf-caption {font-style:italic;font-size:small;}
-        img {align:left;}
         '''
 
-
+    def print_version(self, url):
+        url = url.split('/')
+        url[-1] = 'v_detail_tab_print,'+url[-1]
+        url = '/'.join(url)
+        return url

recipes/ilsole24ore.recipe

@@ -1,71 +1,65 @@
-#!/usr/bin/env  python
+__author__    = 'Marco Saraceno'
-__license__   = 'GPL v3'
+__copyright__ = '2010, Marco Saraceno <marcosaraceno at gmail.com>'
-__author__    = 'Lorenzo Vigentini & Edwin van Maastrigt'
+description   = 'Italian daily newspaper - v 1.1 (Mar14,2011)'
-__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com> and Edwin van Maastrigt <evanmaastrigt at gmail.com>'
-__description__ = 'Financial news daily paper - v1.02 (30, January 2010)'
 
 '''
-http://www.ilsole24ore.com/
+http://www.ilsole24ore.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
+class IlSole24Ore(BasicNewsRecipe):
+    __author__        = 'Marco Saraceno'
+    description   = 'Italian financial daily newspaper'
 
-class ilsole24Ore(BasicNewsRecipe):
+    cover_url      = 'http://www.shopping24.ilsole24ore.com/ProductRelated/rds/img/logo_sole.gif'
-    author        = 'Lorenzo Vigentini & Edwin van Maastrigt'
+    title          = u'Il Sole 24 Ore'
-    description   = 'Financial news daily paper'
+    publisher      = 'Gruppo editoriale GRUPPO 24ORE'
-
+    category       = 'News, politics, culture, economy, financial, Italian'
-    cover_url      = 'http://www.ilsole24ore.com/img2007/print_header.gif'
-
-    title          = u'il Sole 24 Ore New'
-    publisher      = 'italiaNews'
-    category       = 'News, finance, economy, politics'
 
     language       = 'it'
     timefmt        = '[%a, %d %b, %Y]'
 
     oldest_article = 2
-    max_articles_per_feed = 50
+    max_articles_per_feed = 100
     use_embedded_content  = False
+    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
+
+
+    remove_tags = [
+                            dict(name='div', attrs={'class':['header','titolo']}),
+                            dict(name='table', attrs={'class':['footer1024','footerdown']}),
+                            ]
 
-    remove_javascript     = True
-    no_stylesheets        = True
 
     def get_article_url(self, article):
-        return article.get('id', article.get('guid', None))
+       link = article.get('link', None)
-
-    def print_version(self, url):
-        link, sep, params = url.rpartition('?')
        if link is None:
-           return link.replace('_1.php', '_php')
+           return article
-        return link.replace('.shtml', '_PRN.shtml')
+       if link.split('/')[-1]=="story01.htm":
-
+           link=link.split('/')[-2]
-    keep_only_tags     = [
+           a=['0B','0C','0D','0E','0F','0G','0N'  ,'0L0S','0A']
-                            dict(name='div', attrs={'class':'txt'})
+           b=['.' ,'/' ,'?' ,'-' ,'=' ,'&' ,'.com','www.','0']
-                        ]
+           for i in range(0,len(a)):
-#    remove_tags = [dict(name='br')]
+              link=link.replace(a[i],b[i])
+           link="http://"+link
+       return link
 
     feeds = [
-                       (u'Prima pagina', u'http://www.ilsole24ore.com/rss/primapagina.xml'),
+                  (u'Notizie Italia', u'http://www.ilsole24ore.com/rss/notizie/italia.xml'),
-                       (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-tributi.xml'),
+                  (u'Notizie Europa', u'http://www.ilsole24ore.com/rss/notizie/europa.xml'),
-                       (u'Finanza e mercati', u'http://www.ilsole24ore.com/rss/finanza-mercati.xml'),
+                  (u'Notizie USA', u'http://www.ilsole24ore.com/rss/notizie/usa.xml'),
-                       (u'Economia e lavoro', u'http://www.ilsole24ore.com/rss/economia-lavoro.xml'),
+                  (u'Notizie Americhe', u'http://www.ilsole24ore.com/rss/notizie/americhe.xml'),
-                       (u'Italia', u'http://www.ilsole24ore.com/rss/italia.xml'),
+                  (u'Notizie Medio Oriente e Africa', u'http://www.ilsole24ore.com/rss/notizie/medio-oriente-e-africa.xml'),
-                       (u'Mondo', u'http://www.ilsole24ore.com/rss/mondo.xml'),
+                  (u'Notizie Asia e Oceania', u'http://www.ilsole24ore.com/rss/notizie/asia-e-oceania.xml'),
-                       (u'Tecnologia e business', u'http://www.ilsole24ore.com/rss/tecnologia-business.xml'),
+                  (u'Commenti', u'http://www.ilsole24ore.com/rss/commenti-e-idee.xml'),
-                       (u'Cultura e tempo libero', u'http://www.ilsole24ore.com/rss/tempolibero-cultura.xml'),
+                  (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-e-tributi.xml'),
-                       (u'Sport', u'http://www.ilsole24ore.com/rss/sport.xml'),
+                  (u'Finanza', u'http://www.ilsole24ore.com/rss/finanza-e-mercati.xml'),
-                       (u'Professionisti 24', u'http://www.ilsole24ore.com/rss/prof_home.xml'),
+                  (u'Economia', u'http://www.ilsole24ore.com/rss/economia.xml'),
-                       (u'Ambiente e Sicurezza',u'http://www.ilsole24ore.com/rss/prof_as.xml')
+                  (u'Tecnologia', u'http://www.ilsole24ore.com/rss/tecnologie.xml'),
+                  (u'Cultura', u'http://www.ilsole24ore.com/rss/cultura.xml'),
                   ]
 
-    extra_css = '''
+    def print_version(self, url):
-                html, body, table, tr, td, h1, h2, h3, h4, h5, h6, p, a, span, br, img {margin:0;padding:0;border:0;font-size:12px;font-family:"Georgia","Times New Roman";}
+          return url.replace('.shtml', '_PRN.shtml')
-                .linkHighlight {color:#0292c6;}
-                .txt {border-bottom:1px solid #7c7c7c;padding-bottom:20px};text-align:justify;font-family:"serif"}
-                .txt p {line-height:18px;}
-                .txt span {line-height:22px;}
-                .title h3 {color:#7b7b7b;}
-                .title h4 {color:#08526e;font-size:26px;font-family:"Times New Roman";font-weight:normal;}
-                '''
 

recipes/newsweek.recipe

@@ -1,4 +1,3 @@
-import string
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Newsweek(BasicNewsRecipe):
@@ -11,7 +10,6 @@ class Newsweek(BasicNewsRecipe):
     no_stylesheets = True
 
     BASE_URL = 'http://www.newsweek.com'
-    INDEX = BASE_URL+'/topics.html'
 
     keep_only_tags = dict(name='article', attrs={'class':'article-text'})
     remove_tags = [dict(attrs={'data-dartad':True})]
@@ -23,11 +21,14 @@ class Newsweek(BasicNewsRecipe):
         return soup
 
     def newsweek_sections(self):
-        soup = self.index_to_soup(self.INDEX)
+        return [
-        for a in soup.findAll('a', title='Primary tag', href=True):
+                ('Nation', 'http://www.newsweek.com/tag/nation.html'),
-            yield (string.capitalize(self.tag_to_string(a)),
+                ('Society', 'http://www.newsweek.com/tag/society.html'),
-                    self.BASE_URL+a['href'])
+                ('Culture', 'http://www.newsweek.com/tag/culture.html'),
-
+                ('World', 'http://www.newsweek.com/tag/world.html'),
+                ('Politics', 'http://www.newsweek.com/tag/politics.html'),
+                ('Business', 'http://www.newsweek.com/tag/business.html'),
+        ]
 
     def newsweek_parse_section_page(self, soup):
         for article in soup.findAll('article', about=True,

recipes/tabu.recipe

@@ -37,10 +37,12 @@ class TabuRo(BasicNewsRecipe):
                      ]
 
     remove_tags = [
-            dict(name='div', attrs={'class':'asemanatoare'})
+            dict(name='div', attrs={'class':'asemanatoare'}),
+			 dict(name='div', attrs={'class':'social'})
                            ]
 
     remove_tags_after = [
+					  dict(name='div', attrs={'class':'social'}),
                       dict(name='div', attrs={'id':'comments'}),
                       dict(name='div', attrs={'class':'asemanatoare'})
                         ]

recipes/the_journal.recipe

@@ -0,0 +1,26 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011 Phil Burns'
+'''
+TheJournal.ie
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheJournal(BasicNewsRecipe):
+
+    __author_              = 'Phil Burns'
+    title                  = u'TheJournal.ie'
+    oldest_article         = 1
+    max_articles_per_feed  = 100
+    encoding               = 'utf8'
+    language               = 'en_IE'
+    timefmt                = ' (%A, %B %d, %Y)'
+
+    no_stylesheets         = True
+    remove_tags            = [dict(name='div', attrs={'class':'footer'}),
+                           dict(name=['script', 'noscript'])]
+
+    extra_css              = 'p, div { margin: 0pt; border: 0pt; text-indent: 0.5em }'
+
+    feeds                  = [
+                          (u'Latest News', u'http://www.thejournal.ie/feed/')]

resources/template-functions.json

@@ -5,6 +5,7 @@
     "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n    i = 0\n    res = ''\n    for i in range(0, len(args)):\n        res += args[i]\n    return res\n", 
     "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n    return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", 
     "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n    if val:\n        return val\n    else:\n        return value_if_empty\n", 
+    "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n    if mi.book_size is not None:\n        try:\n            return str(mi.book_size)\n        except:\n            pass\n    return ''\n", 
     "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n    if not val:\n        return ''\n    vals = [v.strip() for v in val.split(',')]\n    for v in vals:\n        if v.startswith(key+':'):\n            return v[len(key)+1:]\n    return ''\n", 
     "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return formatter.get_value(name, [], kwargs)\n", 
     "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    return unicode(x - y)\n", 
@@ -25,9 +26,9 @@
     "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return capitalize(val)\n", 
     "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n    return unicode(len(val.split(sep)))\n", 
     "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n    return val.lower()\n", 
-    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
-    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
     "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    v = strcmp(x, y)\n    if v < 0:\n        return lt\n    if v == 0:\n        return eq\n    return gt\n", 
+    "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n    if (len(args) % 2) != 1:\n        raise ValueError(_('switch requires an odd number of arguments'))\n    i = 0\n    while i < len(args):\n        if i + 1 >= len(args):\n            return args[i]\n        if re.search(args[i], val):\n            return args[i+1]\n        i += 2\n", 
+    "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n    locals[target] = value\n    return value\n", 
     "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n    return unicode(getattr(mi, name, None))\n", 
     "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n    x = float(x if x else 0)\n    y = float(y if y else 0)\n    if x < y:\n        return lt\n    if x == y:\n        return eq\n    return gt\n"
 }

setup.py

@@ -15,9 +15,9 @@ from setup import prints, get_warnings
 
 def check_version_info():
     vi = sys.version_info
-    if vi[0] == 2 and vi[1] > 5:
+    if vi[0] == 2 and vi[1] > 6:
         return None
-    return 'calibre requires python >= 2.6'
+    return 'calibre requires python >= 2.7 and < 3'
 
 def option_parser():
     parser = optparse.OptionParser()

setup/__init__.py

@@ -24,8 +24,10 @@ def initialize_constants():
     global __version__, __appname__, modules, functions, basenames, scripts
 
     src = open('src/calibre/constants.py', 'rb').read()
-    __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
+    nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src)
-    __appname__ = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
+    __version__ = '%s.%s.%s'%(nv.group(1), nv.group(2), nv.group(3))
+    __appname__ = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]',
+            src).group(2)
     epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).\
             search(open('src/calibre/linux.py', 'rb').read()).group(1)
     entry_points = eval(epsrc, {'__appname__': __appname__})

setup/installer/windows/freeze.py

@@ -13,7 +13,8 @@ from setup import Command, modules, functions, basenames, __version__, \
 from setup.build_environment import msvc, MT, RC
 from setup.installer.windows.wix import WixMixIn
 
-QT_DIR = 'Q:\\Qt\\4.7.1'
+OPENSSL_DIR = r'Q:\openssl'
+QT_DIR = 'Q:\\Qt\\4.7.2'
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
 LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
@@ -108,6 +109,8 @@ class Win32Freeze(Command, WixMixIn):
         self.dll_dir = self.j(self.base, 'DLLs')
         shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
                 ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
+        for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
+            shutil.copy2(x, self.dll_dir)
         for x in QT_DLLS:
             x += '4.dll'
             if not x.startswith('phonon'): x = 'Qt'+x

setup/installer/windows/notes.rst

@@ -53,12 +53,25 @@ SQLite
 
 Put sqlite3*.h from the sqlite windows amlgamation in ~/sw/include
 
+OpenSSL
+--------
+
+First install ActiveState Perl if you dont already have perl in windows
+Download and untar the openssl tarball, follow the instructions in INSTALL.W32 (use no-asm)
+to install use prefix q:\openssl
+
+perl Configure VC-WIN32 no-asm enable-static-engine --prefix=Q:/openssl
+ms\do_ms.bat
+nmake -f ms\ntdll.mak
+nmake -f ms\ntdll.mak test
+nmake -f ms\ntdll.mak install
+
 Qt
 --------
 
 Extract Qt sourcecode to C:\Qt\4.x.x. Run configure and make::
 
-    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs && nmake
+    configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake
 
 SIP
 -----

src/calibre/__init__.py

@@ -3,11 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import uuid, sys, os, re, logging, time, random, \
+import sys, os, re, time, random, __builtin__, warnings
-       __builtin__, warnings, multiprocessing
-from contextlib import closing
-from urllib import getproxies
-from urllib2 import unquote as urllib2_unquote
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
@@ -16,25 +12,51 @@ from functools import partial
 warnings.simplefilter('ignore', DeprecationWarning)
 
 
-from calibre.constants import iswindows, isosx, islinux, isfreebsd, isfrozen, \
+from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen,
-                              terminal_controller, preferred_encoding, \
+                              preferred_encoding, __appname__, __version__, __author__,
-                              __appname__, __version__, __author__, \
+                              win32event, win32api, winerror, fcntl,
-                              win32event, win32api, winerror, fcntl, \
+                              filesystem_encoding, plugins, config_dir)
-                              filesystem_encoding, plugins, config_dir
+from calibre.startup import winutil, winutilerror
-from calibre.startup import winutil, winutilerror, guess_type
 
-if islinux and not getattr(sys, 'frozen', False):
+if False and islinux and not getattr(sys, 'frozen', False):
-    # Imported before PyQt4 to workaround PyQt4 util-linux conflict on gentoo
+    # Imported before PyQt4 to workaround PyQt4 util-linux conflict discovered on gentoo
+    # See http://bugs.gentoo.org/show_bug.cgi?id=317557
+    # Importing uuid is slow so get rid of this at some point, maybe in a few
+    # years when even Debian has caught up
+    # Also remember to remove it from site.py in the binary builds
+    import uuid
     uuid.uuid4()
 
 if False:
     # Prevent pyflakes from complaining
     winutil, winutilerror, __appname__, islinux, __version__
-    fcntl, win32event, isfrozen, __author__, terminal_controller
+    fcntl, win32event, isfrozen, __author__
-    winerror, win32api, isfreebsd, guess_type
+    winerror, win32api, isfreebsd
 
-import cssutils
+_mt_inited = False
-cssutils.log.setLevel(logging.WARN)
+def _init_mimetypes():
+    global _mt_inited
+    import mimetypes
+    mimetypes.init([P('mime.types')])
+    _mt_inited = True
+
+def guess_type(*args, **kwargs):
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.guess_type(*args, **kwargs)
+
+def guess_all_extensions(*args, **kwargs):
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.guess_all_extensions(*args, **kwargs)
+
+def get_types_map():
+    import mimetypes
+    if not _mt_inited:
+        _init_mimetypes()
+    return mimetypes.types_map
 
 def to_unicode(raw, encoding='utf-8', errors='strict'):
     if isinstance(raw, unicode):
@@ -182,6 +204,7 @@ class CommandLineError(Exception):
     pass
 
 def setup_cli_handlers(logger, level):
+    import logging
     if os.environ.get('CALIBRE_WORKER', None) is not None and logger.handlers:
         return
     logger.setLevel(level)
@@ -243,6 +266,7 @@ def extract(path, dir):
     extractor(path, dir)
 
 def get_proxies(debug=True):
+    from urllib import getproxies
     proxies = getproxies()
     for key, proxy in list(proxies.items()):
         if not proxy or '..' in proxy:
@@ -386,6 +410,7 @@ class StreamReadWrapper(object):
 
 def detect_ncpus():
     """Detects the number of effective CPUs in the system"""
+    import multiprocessing
     ans = -1
     try:
         ans = multiprocessing.cpu_count()
@@ -550,6 +575,9 @@ def get_download_filename(url, cookie_file=None):
     '''
     Get a local filename for a URL using the content disposition header
     '''
+    from contextlib import closing
+    from urllib2 import unquote as urllib2_unquote
+
     filename = ''
 
     br = browser()
@@ -679,4 +707,3 @@ main()
     ipshell()
     sys.argv = old_argv
 
-

src/calibre/constants.py

@@ -1,28 +1,32 @@
+from future_builtins import map
+
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
-__appname__   = 'calibre'
+__appname__   = u'calibre'
-__version__   = '0.7.56'
+numeric_version = (0, 7, 57)
-__author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
+__version__   = u'.'.join(map(unicode, numeric_version))
-
+__author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
-import re, importlib
-_ver = __version__.split('.')
-_ver = [int(re.search(r'(\d+)', x).group(1)) for x in _ver]
-numeric_version = tuple(_ver)
 
 '''
 Various run time constants.
 '''
 
-import sys, locale, codecs, os
+import sys, locale, codecs, os, importlib, collections
+
+_tc = None
+def terminal_controller():
+    global _tc
+    if _tc is None:
         from calibre.utils.terminfo import TerminalController
+        _tc = TerminalController(sys.stdout)
+    return _tc
 
-terminal_controller = TerminalController(sys.stdout)
+_plat = sys.platform.lower()
-
+iswindows = 'win32' in _plat or 'win64' in _plat
-iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower()
+isosx     = 'darwin' in _plat
-isosx     = 'darwin' in sys.platform.lower()
 isnewosx  = isosx and getattr(sys, 'new_app_bundle', False)
-isfreebsd = 'freebsd' in sys.platform.lower()
+isfreebsd = 'freebsd' in _plat
 islinux   = not(iswindows or isosx or isfreebsd)
 isfrozen  = hasattr(sys, 'frozen')
 isunix = isosx or islinux
@@ -41,6 +45,7 @@ fcntl      = None if iswindows else importlib.import_module('fcntl')
 filesystem_encoding = sys.getfilesystemencoding()
 if filesystem_encoding is None: filesystem_encoding = 'utf-8'
 
+
 DEBUG = False
 
 def debug():
@@ -48,15 +53,12 @@ def debug():
     DEBUG = True
 
 # plugins {{{
-plugins = None
-if plugins is None:
-    # Load plugins
-    def load_plugins():
-        plugins = {}
-        plugin_path = sys.extensions_location
-        sys.path.insert(0, plugin_path)
 
-        for plugin in [
+class Plugins(collections.Mapping):
+
+    def __init__(self):
+        self._plugins = {}
+        plugins = [
                 'pictureflow',
                 'lzx',
                 'msdes',
@@ -70,19 +72,44 @@ if plugins is None:
                 'chm_extra',
                 'icu',
                 'speedup',
-            ] + \
+            ]
-                    (['winutil'] if iswindows else []) + \
+        if iswindows:
-                    (['usbobserver'] if isosx else []):
+            plugins.append('winutil')
+        if isosx:
+            plugins.append('usbobserver')
+        self.plugins = frozenset(plugins)
+
+    def load_plugin(self, name):
+        if name in self._plugins:
+            return
+        sys.path.insert(0, sys.extensions_location)
         try:
-                p, err = importlib.import_module(plugin), ''
+            p, err = importlib.import_module(name), ''
         except Exception as err:
             p = None
             err = str(err)
-            plugins[plugin] = (p, err)
+        self._plugins[name] = (p, err)
-        sys.path.remove(plugin_path)
+        sys.path.remove(sys.extensions_location)
-        return plugins
 
-    plugins = load_plugins()
+    def __iter__(self):
+        return iter(self.plugins)
+
+    def __len__(self):
+        return len(self.plugins)
+
+    def __contains__(self, name):
+        return name in self.plugins
+
+    def __getitem__(self, name):
+        if name not in self.plugins:
+            raise KeyError('No plugin named %r'%name)
+        self.load_plugin(name)
+        return self._plugins[name]
+
+
+plugins = None
+if plugins is None:
+    plugins = Plugins()
 # }}}
 
 # config_dir {{{

src/calibre/customize/builtins.py

@@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \
 from calibre.constants import numeric_version
 from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
-from calibre.ebooks.oeb.base import OEB_IMAGES
 from calibre.utils.config import test_eight_code
 
 # To archive plugins {{{
@@ -98,6 +97,8 @@ class TXT2TXTZ(FileTypePlugin):
     on_import = True
 
     def _get_image_references(self, txt, base_dir):
+        from calibre.ebooks.oeb.base import OEB_IMAGES
+
         images = []
 
         # Textile

src/calibre/customize/ui.py

@@ -22,6 +22,11 @@ from calibre.utils.config import make_config_dir, Config, ConfigProxy, \
 from calibre.ebooks.epub.fix import ePubFixer
 from calibre.ebooks.metadata.sources.base import Source
 
+builtin_names = frozenset([p.name for p in builtin_plugins])
+
+class NameConflict(ValueError):
+    pass
+
 def _config():
     c = Config('customize')
     c.add_opt('plugins', default={}, help=_('Installed plugins'))
@@ -355,6 +360,9 @@ def set_file_type_metadata(stream, mi, ftype):
 def add_plugin(path_to_zip_file):
     make_config_dir()
     plugin = load_plugin(path_to_zip_file)
+    if plugin.name in builtin_names:
+        raise NameConflict(
+            'A builtin plugin with the name %r already exists' % plugin.name)
     plugin = initialize_plugin(plugin, path_to_zip_file)
     plugins = config['plugins']
     zfp = os.path.join(plugin_dir, plugin.name+'.zip')
@@ -506,7 +514,11 @@ def initialize_plugin(plugin, path_to_zip_file):
 def initialize_plugins():
     global _initialized_plugins
     _initialized_plugins = []
-    for zfp in list(config['plugins'].values()) + builtin_plugins:
+    conflicts = [name for name in config['plugins'] if name in
+            builtin_names]
+    for p in conflicts:
+        remove_plugin(p)
+    for zfp in list(config['plugins'].itervalues()) + builtin_plugins:
         try:
             try:
                 plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp

src/calibre/debug.py

@@ -106,7 +106,7 @@ def migrate(old, new):
     from calibre.library.database import LibraryDatabase
     from calibre.library.database2 import LibraryDatabase2
     from calibre.utils.terminfo import ProgressBar
-    from calibre import terminal_controller
+    from calibre.constants import terminal_controller
     class Dummy(ProgressBar):
         def setLabelText(self, x): pass
         def setAutoReset(self, y): pass
@@ -119,7 +119,7 @@ def migrate(old, new):
 
     db = LibraryDatabase(old)
     db2 = LibraryDatabase2(new)
-    db2.migrate_old(db, Dummy(terminal_controller, 'Migrating database...'))
+    db2.migrate_old(db, Dummy(terminal_controller(), 'Migrating database...'))
     prefs['library_path'] = os.path.abspath(new)
     print 'Database migrated to', os.path.abspath(new)
 

src/calibre/devices/kindle/apnx.py

@@ -164,7 +164,7 @@ class APNXBuilder(object):
                 if c == '/':
                     closing = True
                     continue
-                elif c == 'p':
+                elif c in ('d', 'p'):
                     if closing:
                         in_p = False
                     else:

src/calibre/devices/scanner.py

@@ -8,7 +8,7 @@ manner.
 import sys, os, re
 from threading import RLock
 
-from calibre import iswindows, isosx, plugins, islinux
+from calibre.constants import iswindows, isosx, plugins, islinux
 
 osx_scanner = win_scanner = linux_scanner = None
 

src/calibre/ebooks/__init__.py

@@ -7,7 +7,7 @@ Code for the conversion of ebook formats and the reading of metadata
 from various formats.
 '''
 
-import traceback, os
+import traceback, os, re
 from calibre import CurrentDir
 
 class ConversionError(Exception):
@@ -169,3 +169,42 @@ def calibre_cover(title, author_string, series_string=None,
         lines.append(TextLine(series_string, author_size))
     return create_cover_page(lines, I('library.png'), output_format='jpg')
 
+UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$')
+
+def unit_convert(value, base, font, dpi):
+    ' Return value in pts'
+    if isinstance(value, (int, long, float)):
+        return value
+    try:
+        return float(value) * 72.0 / dpi
+    except:
+        pass
+    result = value
+    m = UNIT_RE.match(value)
+    if m is not None and m.group(1):
+        value = float(m.group(1))
+        unit = m.group(2)
+        if unit == '%':
+            result = (value / 100.0) * base
+        elif unit == 'px':
+            result = value * 72.0 / dpi
+        elif unit == 'in':
+            result = value * 72.0
+        elif unit == 'pt':
+            result = value
+        elif unit == 'em':
+            result = value * font
+        elif unit in ('ex', 'en'):
+            # This is a hack for ex since we have no way to know
+            # the x-height of the font
+            font = font
+            result = value * font * 0.5
+        elif unit == 'pc':
+            result = value * 12.0
+        elif unit == 'mm':
+            result = value * 0.04
+        elif unit == 'cm':
+            result = value * 0.40
+    return result
+
+

src/calibre/ebooks/chm/reader.py

@@ -5,8 +5,8 @@ __copyright__  = '2008, Kovid Goyal <kovid at kovidgoyal.net>,' \
                  ' and Alex Bramley <a.bramley at gmail.com>.'
 
 import os, re
-from mimetypes import guess_type as guess_mimetype
 
+from calibre import guess_type as guess_mimetype
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString
 from calibre.constants import iswindows, filesystem_encoding
 from calibre.utils.chm.chm import CHMFile

src/calibre/ebooks/conversion/plumber.py

@@ -14,7 +14,8 @@ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.date import parse_date
 from calibre.utils.zipfile import ZipFile
-from calibre import extract, walk, isbytestring, filesystem_encoding
+from calibre import (extract, walk, isbytestring, filesystem_encoding,
+        get_types_map)
 from calibre.constants import __version__
 
 DEBUG_README=u'''
@@ -875,6 +876,9 @@ OptionRecommendation(name='sr3_replace',
         if self.opts.verbose:
             self.log.filter_level = self.log.DEBUG
         self.flush()
+        import cssutils, logging
+        cssutils.log.setLevel(logging.WARN)
+        get_types_map() # Ensure the mimetypes module is intialized
 
         if self.opts.debug_pipeline is not None:
             self.opts.verbose = max(self.opts.verbose, 4)

src/calibre/ebooks/fb2/fb2ml.py

@@ -10,7 +10,6 @@ Transform OEB content into FB2 markup
 
 from base64 import b64encode
 from datetime import datetime
-from mimetypes import types_map
 import re
 import uuid
 
@@ -18,9 +17,6 @@ from lxml import etree
 
 from calibre import prepare_string_for_xml
 from calibre.constants import __appname__, __version__
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
-from calibre.ebooks.oeb.stylizer import Stylizer
-from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES, OPF
 from calibre.utils.magick import Image
 
 class FB2MLizer(object):
@@ -100,6 +96,7 @@ class FB2MLizer(object):
         return text
 
     def fb2_header(self):
+        from calibre.ebooks.oeb.base import OPF
         metadata = {}
         metadata['title'] = self.oeb_book.metadata.title[0].value
         metadata['appname'] = __appname__
@@ -180,6 +177,8 @@ class FB2MLizer(object):
         return u'</FictionBook>'
 
     def get_cover(self):
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
+
         cover_href = None
 
         # Get the raster cover if it's available.
@@ -213,6 +212,8 @@ class FB2MLizer(object):
         return u''
 
     def get_text(self):
+        from calibre.ebooks.oeb.base import XHTML
+        from calibre.ebooks.oeb.stylizer import Stylizer
         text = ['<body>']
 
         # Create main section if there are no others to create
@@ -248,6 +249,8 @@ class FB2MLizer(object):
         '''
         This function uses the self.image_hrefs dictionary mapping. It is populated by the dump_text function.
         '''
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
+
         images = []
         for item in self.oeb_book.manifest:
             # Don't write the image if it's not referenced in the document's text.
@@ -255,7 +258,7 @@ class FB2MLizer(object):
                 continue
             if item.media_type in OEB_RASTER_IMAGES:
                 try:
-                    if not item.media_type == types_map['.jpeg'] or not item.media_type == types_map['.jpg']:
+                    if item.media_type != 'image/jpeg':
                         im = Image()
                         im.load(item.data)
                         im.set_compression_quality(70)
@@ -344,6 +347,8 @@ class FB2MLizer(object):
 
         @return: List of string representing the XHTML converted to FB2 markup.
         '''
+        from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
+
         # Ensure what we are converting is not a string and that the fist tag is part of the XHTML namespace.
         if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS:
             return []

src/calibre/ebooks/html/input.py

@@ -315,7 +315,8 @@ class HTMLInput(InputFormatPlugin):
         from calibre import guess_type
         from calibre.ebooks.oeb.transforms.metadata import \
             meta_info_to_oeb_metadata
-        import cssutils
+        import cssutils, logging
+        cssutils.log.setLevel(logging.WARN)
         self.OEB_STYLES = OEB_STYLES
         oeb = create_oebbook(log, None, opts, self,
                 encoding=opts.input_encoding, populate=False)

src/calibre/ebooks/html/meta.py

@@ -4,7 +4,6 @@ __copyright__ = '2010, Fabian Grassl <fg@jusmeum.de>'
 __docformat__ = 'restructuredtext en'
 
 
-from calibre.ebooks.oeb.base import namespace, barename, DC11_NS
 
 class EasyMeta(object):
 
@@ -12,6 +11,7 @@ class EasyMeta(object):
         self.meta = meta
 
     def __iter__(self):
+        from calibre.ebooks.oeb.base import namespace, barename, DC11_NS
         meta = self.meta
         for item_name in meta.items:
             for item in meta[item_name]:

src/calibre/ebooks/html/output.py

@@ -12,7 +12,6 @@ from os.path import dirname, abspath, relpath, exists, basename
 from lxml import etree
 from templite import Templite
 
-from calibre.ebooks.oeb.base import element
 from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation
 from calibre import CurrentDir
 from calibre.ptempfile import PersistentTemporaryDirectory
@@ -51,6 +50,7 @@ class HTMLOutput(OutputFormatPlugin):
         '''
         Generate table of contents
         '''
+        from calibre.ebooks.oeb.base import element
         with CurrentDir(output_dir):
             def build_node(current_node, parent=None):
                 if parent is None:

src/calibre/ebooks/htmlz/output.py

@@ -12,7 +12,6 @@ from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
-from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
 
@@ -42,6 +41,8 @@ class HTMLZOutput(OutputFormatPlugin):
     ])
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME
+
         # HTML
         if opts.htmlz_css_type == 'inline':
             from calibre.ebooks.htmlz.oeb2html import OEB2HTMLInlineCSSizer

src/calibre/ebooks/metadata/__init__.py

@@ -6,11 +6,11 @@ __docformat__ = 'restructuredtext en'
 """
 Provides abstraction for metadata reading.writing from a variety of ebook formats.
 """
-import os, mimetypes, sys, re
+import os, sys, re
 from urllib import unquote, quote
 from urlparse import urlparse
 
-from calibre import relpath
+from calibre import relpath, guess_type
 
 from calibre.utils.config import tweaks
 
@@ -118,7 +118,7 @@ class Resource(object):
         self.path = None
         self.fragment = ''
         try:
-            self.mime_type = mimetypes.guess_type(href_or_path)[0]
+            self.mime_type = guess_type(href_or_path)[0]
         except:
             self.mime_type = None
         if self.mime_type is None:

src/calibre/ebooks/metadata/book/base.py

@@ -592,7 +592,7 @@ class Metadata(object):
             elif datatype == 'bool':
                 res = _('Yes') if res else _('No')
             elif datatype == 'rating':
-                res = res/2
+                res = res/2.0
             return (name, unicode(res), orig_res, cmeta)
 
         # convert top-level ids into their value
@@ -626,7 +626,7 @@ class Metadata(object):
             elif datatype == 'datetime':
                 res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy'))
             elif datatype == 'rating':
-                res = res/2
+                res = res/2.0
             return (name, unicode(res), orig_res, fmeta)
 
         return (None, None, None, None)

src/calibre/ebooks/metadata/fb2.py

@@ -5,11 +5,12 @@ __copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 
 '''Read meta information from fb2 files'''
 
-import mimetypes, os
+import os
 from base64 import b64decode
 from lxml import etree
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.chardet import xml_to_unicode
+from calibre import guess_all_extensions
 
 XLINK_NS     = 'http://www.w3.org/1999/xlink'
 def XLINK(name):
@@ -71,7 +72,7 @@ def get_metadata(stream):
             binary = XPath('//fb2:binary[@id="%s"]'%id)(root)
             if binary:
                 mt = binary[0].get('content-type', 'image/jpeg')
-                exts = mimetypes.guess_all_extensions(mt)
+                exts = guess_all_extensions(mt)
                 if not exts:
                     exts = ['.jpg']
                 cdata = (exts[0][1:], b64decode(tostring(binary[0])))

src/calibre/ebooks/metadata/opf2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json
+import re, sys, unittest, functools, os, uuid, glob, cStringIO, json
 from urllib import unquote
 from urlparse import urlparse
 
@@ -20,7 +20,7 @@ from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_is
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import parse_date, isoformat
 from calibre.utils.localization import get_lang
-from calibre import prints
+from calibre import prints, guess_type
 from calibre.utils.cleantext import clean_ascii_chars
 
 class Resource(object): # {{{
@@ -42,7 +42,7 @@ class Resource(object): # {{{
         self.path = None
         self.fragment = ''
         try:
-            self.mime_type = mimetypes.guess_type(href_or_path)[0]
+            self.mime_type = guess_type(href_or_path)[0]
         except:
             self.mime_type = None
         if self.mime_type is None:
@@ -1000,7 +1000,7 @@ class OPF(object): # {{{
                 for t in ('cover', 'other.ms-coverimage-standard', 'other.ms-coverimage'):
                     for item in self.guide:
                         if item.type.lower() == t:
-                            self.create_manifest_item(item.href(), mimetypes.guess_type(path)[0])
+                            self.create_manifest_item(item.href(), guess_type(path)[0])
 
         return property(fget=fget, fset=fset)
 

src/calibre/ebooks/metadata/sources/base.py

@@ -25,6 +25,7 @@ msprefs.defaults['max_tags'] = 20
 msprefs.defaults['wait_after_first_identify_result'] = 30 # seconds
 msprefs.defaults['wait_after_first_cover_result'] = 60 # seconds
 msprefs.defaults['swap_author_names'] = False
+msprefs.defaults['fewer_tags'] = True
 
 # Google covers are often poor quality (scans/errors) but they have high
 # resolution, so they trump covers from better sources. So make sure they

src/calibre/ebooks/metadata/sources/identify.py

@@ -42,6 +42,10 @@ class Worker(Thread):
             self.log.exception('Plugin', self.plugin.name, 'failed')
         self.plugin.dl_time_spent = time.time() - start
 
+    @property
+    def name(self):
+        return self.plugin.name
+
 def is_worker_alive(workers):
     for w in workers:
         if w.is_alive():
@@ -216,7 +220,7 @@ class ISBNMerge(object):
 
         # We assume the smallest set of tags has the least cruft in it
         ans.tags = self.length_merge('tags', results,
-                null_value=ans.tags)
+                null_value=ans.tags, shortest=msprefs['fewer_tags'])
 
         # We assume the longest series has the most info in it
         ans.series = self.length_merge('series', results,
@@ -348,7 +352,11 @@ def identify(log, abort, # {{{
 
         if (first_result_at is not None and time.time() - first_result_at >
                 wait_time):
-            log('Not waiting any longer for more results')
+            log.warn('Not waiting any longer for more results. Still running'
+                    ' sources:')
+            for worker in workers:
+                if worker.is_alive():
+                    log.debug('\t' + worker.name)
             abort.set()
             break
 
@@ -382,7 +390,11 @@ def identify(log, abort, # {{{
             log(plog)
         log('\n'+'*'*80)
 
+        dummy = Metadata(_('Unknown'))
         for i, result in enumerate(presults):
+            for f in plugin.prefs['ignore_fields']:
+                if ':' not in f:
+                    setattr(result, f, getattr(dummy, f))
             result.relevance_in_source = i
             result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable
                     and plugin.get_cached_cover_url(result.identifiers) is not

src/calibre/ebooks/metadata/sources/overdrive.py

@@ -3,13 +3,13 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__copyright__ = '2011, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
 '''
 Fetch metadata using Overdrive Content Reserve
 '''
-import re, random, mechanize, copy
+import re, random, mechanize, copy, json
 from threading import RLock
 from Queue import Queue, Empty
 
@@ -17,13 +17,12 @@ from lxml import html
 from lxml.html import soupparser
 
 from calibre.ebooks.metadata import check_isbn
-from calibre.ebooks.metadata.sources.base import Source
+from calibre.ebooks.metadata.sources.base import Source, Option
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.library.comments import sanitize_comments_html
 
 ovrdrv_data_cache = {}
-cover_url_cache = {}
 cache_lock = RLock()
 base_url = 'http://search.overdrive.com/'
 
@@ -41,9 +40,17 @@ class OverDrive(Source):
     supports_gzip_transfer_encoding = False
     cached_cover_url_is_reliable = True
 
-    def __init__(self, *args, **kwargs):
+    options = (
-       Source.__init__(self, *args, **kwargs)
+            Option('get_full_metadata', 'bool', False,
-       self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn', 'language']
+                _('Download all metadata (slow)'),
+                _('Enable this option to gather all metadata available from Overdrive.')),
+            )
+
+    config_help_message = '<p>'+_('Additional metadata can be taken from Overdrive\'s book detail'
+            ' page. This includes a limited set of tags used by libraries, comments, language,'
+            ' and the ebook ISBN. Collecting this data is disabled by default due to the extra'
+            ' time required. Check the download all metadata option below to'
+            ' enable downloading this data.')
 
     def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
             identifiers={}, timeout=30):
@@ -51,7 +58,7 @@ class OverDrive(Source):
         isbn = identifiers.get('isbn', None)
 
         br = self.browser
-        ovrdrv_data = self.to_ovrdrv_data(br, title, authors, ovrdrv_id)
+        ovrdrv_data = self.to_ovrdrv_data(br, log, title, authors, ovrdrv_id)
         if ovrdrv_data:
             title = ovrdrv_data[8]
             authors = ovrdrv_data[6]
@@ -59,11 +66,13 @@ class OverDrive(Source):
             self.parse_search_results(ovrdrv_data, mi)
             if ovrdrv_id is None:
                 ovrdrv_id = ovrdrv_data[7]
+
+            if self.prefs['get_full_metadata']:
+                self.get_book_detail(br, ovrdrv_data[1], mi, ovrdrv_id, log)
+
             if isbn is not None:
                 self.cache_isbn_to_identifier(isbn, ovrdrv_id)
 
-            self.get_book_detail(br, ovrdrv_data[1], mi, ovrdrv_id, log)
-
             result_queue.put(mi)
 
         return None
@@ -100,9 +109,11 @@ class OverDrive(Source):
 
         ovrdrv_id = identifiers.get('overdrive', None)
         br = self.browser
-        referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id
         req = mechanize.Request(cached_url)
+        if ovrdrv_id is not None:
+            referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id
             req.add_header('referer', referer)
+
         log('Downloading cover from:', cached_url)
         try:
             cdata = br.open_novisit(req, timeout=timeout).read()
@@ -175,7 +186,7 @@ class OverDrive(Source):
 
         br.set_cookiejar(clean_cj)
 
-    def overdrive_search(self, br, q, title, author):
+    def overdrive_search(self, br, log, q, title, author):
         # re-initialize the cookiejar to so that it's clean
         clean_cj = mechanize.CookieJar()
         br.set_cookiejar(clean_cj)
@@ -193,7 +204,8 @@ class OverDrive(Source):
         else:
             initial_q = ' '.join(author_tokens)
             xref_q = '+'.join(title_tokens)
-
+        #log.error('Initial query is %s'%initial_q)
+        #log.error('Cross reference query is %s'%xref_q)
         q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q
         query = '{"szKeyword":"'+initial_q+'"}'
 
@@ -228,7 +240,7 @@ class OverDrive(Source):
     def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None):
         close_matches = []
         raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw)
-        results = eval(raw)
+        results = json.loads(raw)
         #print results
         # The search results are either from a keyword search or a multi-format list from a single ID,
         # sort through the results for closest match/format
@@ -244,7 +256,7 @@ class OverDrive(Source):
                 else:
                     creators = creators.split(', ')
                     # if an exact match in a preferred format occurs
-                    if (author and creators[0] == author[0]) and od_title == title and int(formatid) in [1, 50, 410, 900]:
+                    if (author and creators[0] == author[0]) and od_title == title and int(formatid) in [1, 50, 410, 900] and thumbimage:
                         return self.format_results(reserveid, od_title, subtitle, series, publisher,
                                 creators, thumbimage, worldcatlink, formatid)
                     else:
@@ -302,16 +314,16 @@ class OverDrive(Source):
         return self.sort_ovrdrv_results(raw, None, None, None, ovrdrv_id)
 
 
-    def find_ovrdrv_data(self, br, title, author, isbn, ovrdrv_id=None):
+    def find_ovrdrv_data(self, br, log, title, author, isbn, ovrdrv_id=None):
         q = base_url
         if ovrdrv_id is None:
-           return self.overdrive_search(br, q, title, author)
+           return self.overdrive_search(br, log, q, title, author)
         else:
            return self.overdrive_get_record(br, q, ovrdrv_id)
 
 
 
-    def to_ovrdrv_data(self, br, title=None, author=None, ovrdrv_id=None):
+    def to_ovrdrv_data(self, br, log, title=None, author=None, ovrdrv_id=None):
         '''
         Takes either a title/author combo or an Overdrive ID.  One of these
         two must be passed to this function.
@@ -324,10 +336,10 @@ class OverDrive(Source):
             elif ans is False:
                 return None
             else:
-                ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id)
+                ovrdrv_data = self.find_ovrdrv_data(br, log, title, author, ovrdrv_id)
         else:
             try:
-                ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id)
+                ovrdrv_data = self.find_ovrdrv_data(br, log, title, author, ovrdrv_id)
             except:
                 import traceback
                 traceback.print_exc()
@@ -390,9 +402,14 @@ class OverDrive(Source):
 
         if pub_date:
             from calibre.utils.date import parse_date
+            try:
                 mi.pubdate = parse_date(pub_date[0].strip())
+            except:
+                pass
         if lang:
-            mi.language = lang[0].strip()
+            lang = lang[0].strip().lower()
+            mi.language = {'english':'en', 'french':'fr', 'german':'de',
+                    'spanish':'es'}.get(lang, None)
 
         if ebook_isbn:
             #print "ebook isbn is "+str(ebook_isbn[0])
@@ -436,4 +453,3 @@ if __name__ == '__main__':
                     authors_test(['Agatha Christie'])]
             ),
     ])
-

src/calibre/ebooks/mobi/reader.py

@@ -20,7 +20,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import parse_date
 from calibre.utils.cleantext import clean_ascii_chars
 from calibre.ptempfile import TemporaryDirectory
-from calibre.ebooks import DRMError
+from calibre.ebooks import DRMError, unit_convert
 from calibre.ebooks.chardet import ENCODING_PATS
 from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.mobi.huffcdic import HuffReader
@@ -258,6 +258,8 @@ class MobiReader(object):
                 }
                 ''')
         self.tag_css_rules = {}
+        self.left_margins = {}
+        self.text_indents = {}
 
         if hasattr(filename_or_stream, 'read'):
             stream = filename_or_stream
@@ -567,9 +569,21 @@ class MobiReader(object):
                     elif tag.tag == 'img':
                         tag.set('width', width)
                     else:
-                        styles.append('text-indent: %s' % self.ensure_unit(width))
+                        ewidth = self.ensure_unit(width)
+                        styles.append('text-indent: %s' % ewidth)
+                        try:
+                            ewidth_val = unit_convert(ewidth, 12, 500, 166)
+                            self.text_indents[tag] = ewidth_val
+                        except:
+                            pass
                         if width.startswith('-'):
                             styles.append('margin-left: %s' % self.ensure_unit(width[1:]))
+                            try:
+                                ewidth_val = unit_convert(ewidth[1:], 12, 500, 166)
+                                self.left_margins[tag] = ewidth_val
+                            except:
+                                pass
+
             if attrib.has_key('align'):
                 align = attrib.pop('align').strip()
                 if align:
@@ -661,6 +675,26 @@ class MobiReader(object):
             if hasattr(parent, 'remove'):
                 parent.remove(tag)
 
+    def get_left_whitespace(self, tag):
+
+        def whitespace(tag):
+            lm = ti = 0.0
+            if tag.tag == 'p':
+                ti = unit_convert('1.5em', 12, 500, 166)
+            if tag.tag == 'blockquote':
+                lm = unit_convert('2em', 12, 500, 166)
+            lm = self.left_margins.get(tag, lm)
+            ti = self.text_indents.get(tag, ti)
+            return lm + ti
+
+        parent = tag
+        ans = 0.0
+        while parent is not None:
+            ans += whitespace(parent)
+            parent = parent.getparent()
+
+        return ans
+
     def create_opf(self, htmlfile, guide=None, root=None):
         mi = getattr(self.book_header.exth, 'mi', self.embedded_mi)
         if mi is None:
@@ -731,16 +765,45 @@ class MobiReader(object):
                             except:
                                 text = ''
                             text = ent_pat.sub(entity_to_unicode, text)
-                            tocobj.add_item(toc.partition('#')[0], href[1:],
+                            item = tocobj.add_item(toc.partition('#')[0], href[1:],
                                 text)
+                            item.left_space = int(self.get_left_whitespace(x))
                             found = True
                     if reached and found and x.get('class', None) == 'mbp_pagebreak':
                         break
             if tocobj is not None:
+                tocobj = self.structure_toc(tocobj)
                 opf.set_toc(tocobj)
 
         return opf, ncx_manifest_entry
 
+    def structure_toc(self, toc):
+        indent_vals = set()
+        for item in toc:
+            indent_vals.add(item.left_space)
+        if len(indent_vals) > 6 or len(indent_vals) < 2:
+            # Too many or too few levels, give up
+            return toc
+        indent_vals = sorted(indent_vals)
+
+        last_found = [None for i in indent_vals]
+
+        newtoc = TOC()
+
+        def find_parent(level):
+            candidates = last_found[:level]
+            for x in reversed(candidates):
+                if x is not None:
+                    return x
+            return newtoc
+
+        for item in toc:
+            level = indent_vals.index(item.left_space)
+            parent = find_parent(level)
+            last_found[level] = parent.add_item(item.href, item.fragment,
+                        item.text)
+
+        return newtoc
 
     def sizeof_trailing_entries(self, data):
         def sizeof_trailing_entry(ptr, psize):

src/calibre/ebooks/oeb/base.py

@@ -8,23 +8,18 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import os, re, uuid, logging
-from mimetypes import types_map
 from collections import defaultdict
 from itertools import count
 from urlparse import urldefrag, urlparse, urlunparse, urljoin
 from urllib import unquote as urlunquote
 
 from lxml import etree, html
-from cssutils import CSSParser, parseString, parseStyle, replaceUrls
+from calibre.constants import filesystem_encoding, __version__
-from cssutils.css import CSSRule
-
-import calibre
-from calibre.constants import filesystem_encoding
 from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.ebooks.conversion.preprocess import CSSPreProcessor
-from calibre import isbytestring, as_unicode
+from calibre import isbytestring, as_unicode, get_types_map
 
 RECOVER_PARSER = etree.XMLParser(recover=True, no_network=True)
 
@@ -179,6 +174,9 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
     If the ``link_repl_func`` returns None, the attribute or
     tag text will be removed completely.
     '''
+    from cssutils import parseString, parseStyle, replaceUrls, log
+    log.setLevel(logging.WARN)
+
     if resolve_base_href:
         resolve_base_href(root)
     for el, attrib, link, pos in iterlinks(root, find_links_in_css=False):
@@ -248,7 +246,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False):
                 el.attrib['style'] = repl
 
 
-
+types_map = get_types_map()
 EPUB_MIME      = types_map['.epub']
 XHTML_MIME     = types_map['.xhtml']
 CSS_MIME       = types_map['.css']
@@ -1075,7 +1073,9 @@ class Manifest(object):
 
 
         def _parse_css(self, data):
-
+            from cssutils.css import CSSRule
+            from cssutils import CSSParser, log
+            log.setLevel(logging.WARN)
             def get_style_rules_from_import(import_rule):
                 ans = []
                 if not import_rule.styleSheet:
@@ -2011,7 +2011,7 @@ class OEBBook(object):
             name='dtb:uid', content=unicode(self.uid))
         etree.SubElement(head, NCX('meta'),
             name='dtb:depth', content=str(self.toc.depth()))
-        generator = ''.join(['calibre (', calibre.__version__, ')'])
+        generator = ''.join(['calibre (', __version__, ')'])
         etree.SubElement(head, NCX('meta'),
             name='dtb:generator', content=generator)
         etree.SubElement(head, NCX('meta'),

src/calibre/ebooks/oeb/profile.py

@@ -1,75 +0,0 @@
-'''
-Device profiles.
-'''
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
-
-from itertools import izip
-
-FONT_SIZES = [('xx-small', 1),
-              ('x-small',  None),
-              ('small',    2),
-              ('medium',   3),
-              ('large',    4),
-              ('x-large',  5),
-              ('xx-large', 6),
-              (None,       7)]
-
-
-class Profile(object):
-    def __init__(self, width, height, dpi, fbase, fsizes):
-        self.width = (float(width) / dpi) * 72.
-        self.height = (float(height) / dpi) * 72.
-        self.dpi = float(dpi)
-        self.fbase = float(fbase)
-        self.fsizes = []
-        for (name, num), size in izip(FONT_SIZES, fsizes):
-            self.fsizes.append((name, num, float(size)))
-        self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
-        self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
-
-
-PROFILES = {
-    'PRS505':
-        Profile(width=584, height=754, dpi=168.451, fbase=12,
-                fsizes=[7.5, 9, 10, 12, 15.5, 20, 22, 24]),
-
-    'MSReader':
-        Profile(width=480, height=652, dpi=96, fbase=13,
-                fsizes=[10, 11, 13, 16, 18, 20, 22, 26]),
-
-    # Not really, but let's pretend
-    'Mobipocket':
-        Profile(width=600, height=800, dpi=96, fbase=18,
-                fsizes=[14, 14, 16, 18, 20, 22, 24, 26]),
-    
-    # No clue on usable screen size; DPI should be good
-    'HanlinV3':
-        Profile(width=584, height=754, dpi=168.451, fbase=16,
-                fsizes=[12, 12, 14, 16, 18, 20, 22, 24]),
-
-    'CybookG3':
-        Profile(width=600, height=800, dpi=168.451, fbase=16,
-                fsizes=[12, 12, 14, 16, 18, 20, 22, 24]),
-
-    'Kindle':
-        Profile(width=525, height=640, dpi=168.451, fbase=16,
-                fsizes=[12, 12, 14, 16, 18, 20, 22, 24]),
-    
-    'Browser':
-        Profile(width=800, height=600, dpi=100.0, fbase=12,
-                fsizes=[5, 7, 9, 12, 13.5, 17, 20, 22, 24])
-    }
-
-
-class Context(object):
-    PROFILES = PROFILES
-    
-    def __init__(self, source, dest):
-        if source in PROFILES:
-            source = PROFILES[source]
-        if dest in PROFILES:
-            dest = PROFILES[dest]
-        self.source = source
-        self.dest = dest

src/calibre/ebooks/oeb/reader.py

@@ -10,11 +10,9 @@ import sys, os, uuid, copy, re, cStringIO
 from itertools import izip
 from urlparse import urldefrag, urlparse
 from urllib import unquote as urlunquote
-from mimetypes import guess_type
 from collections import defaultdict
 
 from lxml import etree
-import cssutils
 
 from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \
     DC_NSES, OPF, xml2text
@@ -30,6 +28,7 @@ from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.utils.localization import get_lang
 from calibre.ptempfile import TemporaryDirectory
 from calibre.constants import __appname__, __version__
+from calibre import guess_type
 
 __all__ = ['OEBReader']
 
@@ -172,6 +171,7 @@ class OEBReader(object):
         return bad
 
     def _manifest_add_missing(self, invalid):
+        import cssutils
         manifest = self.oeb.manifest
         known = set(manifest.hrefs)
         unchecked = set(manifest.values())

src/calibre/ebooks/oeb/stylizer.py

@@ -12,16 +12,17 @@ import os, itertools, re, logging, copy, unicodedata
 from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
-from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
+from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration,
-    CSSValueList, CSSFontFaceRule, cssproperties
+    CSSValueList, CSSFontFaceRule, cssproperties)
 from cssutils import profile as cssprofiles
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError
-
 from calibre import force_unicode
+from calibre.ebooks import unit_convert
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize
-from calibre.ebooks.oeb.profile import PROFILES
+
+cssutils.log.setLevel(logging.WARN)
 
 _html_css_stylesheet = None
 
@@ -121,10 +122,10 @@ class CSSSelector(etree.XPath):
 class Stylizer(object):
     STYLESHEETS = WeakKeyDictionary()
 
-    def __init__(self, tree, path, oeb, opts, profile=PROFILES['PRS505'],
+    def __init__(self, tree, path, oeb, opts, profile=None,
             extra_css='', user_css=''):
         self.oeb, self.opts = oeb, opts
-        self.profile = profile
+        self.profile = opts.input_profile
         self.logger = oeb.logger
         item = oeb.manifest.hrefs[path]
         basename = os.path.basename(path)
@@ -443,7 +444,6 @@ class Stylizer(object):
 
 
 class Style(object):
-    UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$')
     MS_PAT = re.compile(r'^\s*(mso-|panose-|text-underline|tab-interval)')
 
     def __init__(self, element, stylizer):
@@ -507,42 +507,10 @@ class Style(object):
 
     def _unit_convert(self, value, base=None, font=None):
         'Return value in pts'
-        if isinstance(value, (int, long, float)):
-            return value
-        try:
-            return float(value) * 72.0 / self._profile.dpi
-        except:
-            pass
-        result = value
-        m = self.UNIT_RE.match(value)
-        if m is not None and m.group(1):
-            value = float(m.group(1))
-            unit = m.group(2)
-            if unit == '%':
         if base is None:
             base = self.width
-                result = (value / 100.0) * base
-            elif unit == 'px':
-                result = value * 72.0 / self._profile.dpi
-            elif unit == 'in':
-                result = value * 72.0
-            elif unit == 'pt':
-                result = value
-            elif unit == 'em':
         font = font or self.fontSize
-                result = value * font
+        return unit_convert(value, base, font, self._profile.dpi)
-            elif unit in ('ex', 'en'):
-                # This is a hack for ex since we have no way to know
-                # the x-height of the font
-                font = font or self.fontSize
-                result = value * font * 0.5
-            elif unit == 'pc':
-                result = value * 12.0
-            elif unit == 'mm':
-                result = value * 0.04
-            elif unit == 'cm':
-                result = value * 0.40
-        return result
 
     def pt_to_px(self, value):
         return (self._profile.dpi / 72.0) * value

src/calibre/ebooks/oeb/transforms/filenames.py

@@ -9,7 +9,6 @@ import posixpath
 from urlparse import urldefrag, urlparse
 
 from lxml import etree
-import cssutils
 
 from calibre.ebooks.oeb.base import rewrite_links, urlnormalize
 
@@ -25,6 +24,7 @@ class RenameFiles(object): # {{{
         self.renamed_items_map = renamed_items_map
 
     def __call__(self, oeb, opts):
+        import cssutils
         self.log = oeb.logger
         self.opts = opts
         self.oeb = oeb

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -8,8 +8,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
 from urlparse import urldefrag
 
-import cssutils
-
 from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS
 from calibre.ebooks.oeb.base import urlnormalize, iterlinks
 
@@ -23,6 +21,7 @@ class ManifestTrimmer(object):
         return cls()
 
     def __call__(self, oeb, context):
+        import cssutils
         oeb.logger.info('Trimming unused files from manifest...')
         self.opts = context
         used = set()

src/calibre/ebooks/pdb/ereader/writer.py

@@ -21,7 +21,6 @@ except ImportError:
 import cStringIO
 
 from calibre.ebooks.pdb.formatwriter import FormatWriter
-from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
 from calibre.ebooks.pdb.header import PdbHeaderBuilder
 from calibre.ebooks.pml.pmlml import PMLMLizer
 
@@ -135,6 +134,7 @@ class Writer(FormatWriter):
         62-...: Raw image data in 8 bit PNG format.
         '''
         images = []
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
 
         for item in manifest:
             if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys():

src/calibre/ebooks/pml/output.py

@@ -18,7 +18,6 @@ from calibre.customize.conversion import OutputFormatPlugin
 from calibre.customize.conversion import OptionRecommendation
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.zipfile import ZipFile
-from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
 from calibre.ebooks.pml.pmlml import PMLMLizer
 
 class PMLOutput(OutputFormatPlugin):
@@ -60,6 +59,7 @@ class PMLOutput(OutputFormatPlugin):
             pmlz.add_dir(tdir)
 
     def write_images(self, manifest, image_hrefs, out_dir, opts):
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
         for item in manifest:
             if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys():
                 if opts.full_image_depth:

src/calibre/ebooks/pml/pmlml.py

@@ -12,8 +12,6 @@ import re
 
 from lxml import etree
 
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
-from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.pdb.ereader import image_name
 from calibre.ebooks.pml import unipmlcode
 
@@ -110,6 +108,9 @@ class PMLMLizer(object):
         return output
 
     def get_cover_page(self):
+        from calibre.ebooks.oeb.stylizer import Stylizer
+        from calibre.ebooks.oeb.base import XHTML
+
         output = u''
         if 'cover' in self.oeb_book.guide:
             output += '\\m="cover.png"\n'
@@ -125,6 +126,9 @@ class PMLMLizer(object):
         return output
 
     def get_text(self):
+        from calibre.ebooks.oeb.stylizer import Stylizer
+        from calibre.ebooks.oeb.base import XHTML
+
         text = [u'']
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to PML markup...' % item.href)
@@ -214,6 +218,8 @@ class PMLMLizer(object):
         return text
 
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
+        from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
+
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
             return []

src/calibre/ebooks/rb/rbml.py

@@ -11,8 +11,6 @@ Transform OEB content into RB compatible markup.
 import re
 
 from calibre import prepare_string_for_xml
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
-from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.rb import unique_name
 
 TAGS = [
@@ -81,6 +79,8 @@ class RBMLizer(object):
         return output
 
     def get_cover_page(self):
+        from calibre.ebooks.oeb.stylizer import Stylizer
+        from calibre.ebooks.oeb.base import XHTML
         output = u''
         if 'cover' in self.oeb_book.guide:
             if self.name_map.get(self.oeb_book.guide['cover'].href, None):
@@ -109,6 +109,9 @@ class RBMLizer(object):
         return ''.join(toc)
 
     def get_text(self):
+        from calibre.ebooks.oeb.stylizer import Stylizer
+        from calibre.ebooks.oeb.base import XHTML
+
         output = [u'']
         for item in self.oeb_book.spine:
             self.log.debug('Converting %s to RocketBook HTML...' % item.href)
@@ -137,6 +140,8 @@ class RBMLizer(object):
         return text
 
     def dump_text(self, elem, stylizer, page, tag_stack=[]):
+        from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
+
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
             return [u'']

src/calibre/ebooks/rb/writer.py

@@ -18,7 +18,6 @@ import cStringIO
 from calibre.ebooks.rb.rbml import RBMLizer
 from calibre.ebooks.rb import HEADER
 from calibre.ebooks.rb import unique_name
-from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
 from calibre.constants import __appname__, __version__
 
 TEXT_RECORD_SIZE = 4096
@@ -111,6 +110,7 @@ class RBWriter(object):
         return (size, pages)
 
     def _images(self, manifest):
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
         images = []
         used_names = []
 

src/calibre/ebooks/rtf/rtfml.py

@@ -14,9 +14,6 @@ import cStringIO
 
 from lxml import etree
 
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, \
-    OEB_RASTER_IMAGES
-from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.filenames import ascii_text
 from calibre.utils.magick.draw import save_cover_data_to, identify_data
@@ -100,6 +97,8 @@ class RTFMLizer(object):
         return self.mlize_spine()
 
     def mlize_spine(self):
+        from calibre.ebooks.oeb.base import XHTML
+        from calibre.ebooks.oeb.stylizer import Stylizer
         output = self.header()
         if 'titlepage' in self.oeb_book.guide:
             href = self.oeb_book.guide['titlepage'].href
@@ -154,6 +153,8 @@ class RTFMLizer(object):
         return ' }'
 
     def insert_images(self, text):
+        from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
+
         for item in self.oeb_book.manifest:
             if item.media_type in OEB_RASTER_IMAGES:
                 src = os.path.basename(item.href)
@@ -201,6 +202,8 @@ class RTFMLizer(object):
         return text
 
     def dump_text(self, elem, stylizer, tag_stack=[]):
+        from calibre.ebooks.oeb.base import XHTML_NS, namespace, barename
+
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:
             p = elem.getparent()

src/calibre/ebooks/snb/input.py

@@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
 import os, uuid
 
 from calibre.customize.conversion import InputFormatPlugin
-from calibre.ebooks.oeb.base import DirContainer
 from calibre.ebooks.snb.snbfile import SNBFile
 from calibre.ptempfile import TemporaryDirectory
 from calibre.utils.filenames import ascii_filename
@@ -30,6 +29,7 @@ class SNBInput(InputFormatPlugin):
 
     def convert(self, stream, options, file_ext, log,
                 accelerators):
+        from calibre.ebooks.oeb.base import DirContainer
         log.debug("Parsing SNB file...")
         snbFile = SNBFile()
         try:

src/calibre/ebooks/snb/snbfile.py

@@ -5,7 +5,8 @@ __copyright__ = '2010, Li Fanxi <lifanxi@freemindworld.com>'
 __docformat__ = 'restructuredtext en'
 
 import sys, struct, zlib, bz2, os
-from mimetypes import types_map
+
+from calibre import guess_type
 
 class FileStream:
     def IsBinary(self):
@@ -180,7 +181,7 @@ class SNBFile:
                 file = open(os.path.join(path, fname), 'wb')
                 file.write(f.fileBody)
                 file.close()
-                fileNames.append((fname, types_map[ext]))
+                fileNames.append((fname, guess_type('a'+ext)[0]))
         return fileNames
 
     def Output(self, outputFile):

src/calibre/ebooks/snb/snbml.py

@@ -13,8 +13,6 @@ import re
 
 from lxml import etree
 
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
-from calibre.ebooks.oeb.stylizer import Stylizer
 
 def ProcessFileName(fileName):
     # Flat the path
@@ -81,6 +79,8 @@ class SNBMLizer(object):
                     body.append(entity)
 
     def mlize(self):
+        from calibre.ebooks.oeb.base import XHTML
+        from calibre.ebooks.oeb.stylizer import Stylizer
         output = [ u'' ]
         stylizer = Stylizer(self.item.data, self.item.href, self.oeb_book, self.opts, self.opts.output_profile)
         content = unicode(etree.tostring(self.item.data.find(XHTML('body')), encoding=unicode))
@@ -208,6 +208,7 @@ class SNBMLizer(object):
         return text
 
     def dump_text(self, subitems, elem, stylizer, end='', pre=False, li = ''):
+        from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
 
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:

src/calibre/ebooks/txt/output.py

@@ -11,7 +11,6 @@ from lxml import etree
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
-from calibre.ebooks.oeb.base import OEB_IMAGES
 from calibre.ebooks.txt.txtml import TXTMLizer
 from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines
 from calibre.ptempfile import TemporaryDirectory, TemporaryFile
@@ -109,6 +108,7 @@ class TXTZOutput(TXTOutput):
     file_type = 'txtz'
 
     def convert(self, oeb_book, output_path, input_plugin, opts, log):
+        from calibre.ebooks.oeb.base import OEB_IMAGES
         with TemporaryDirectory('_txtz_output') as tdir:
             # TXT
             with TemporaryFile('index.txt') as tf:

src/calibre/ebooks/txt/txtml.py

@@ -12,8 +12,6 @@ import re
 
 from lxml import etree
 
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace
-from calibre.ebooks.oeb.stylizer import Stylizer
 
 BLOCK_TAGS = [
     'div',
@@ -64,6 +62,8 @@ class TXTMLizer(object):
         return self.mlize_spine()
 
     def mlize_spine(self):
+        from calibre.ebooks.oeb.base import XHTML
+        from calibre.ebooks.oeb.stylizer import Stylizer
         output = [u'']
         output.append(self.get_toc())
         for item in self.oeb_book.spine:
@@ -185,6 +185,7 @@ class TXTMLizer(object):
         @stylizer: The style information attached to the element.
         @page: OEB page used to determine absolute urls.
         '''
+        from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace
 
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) != XHTML_NS:

src/calibre/gui2/__init__.py

@@ -4,19 +4,17 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, sys, Queue, threading
 from threading import RLock
 from urllib import unquote
-
+from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt,
-from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \
+                    QByteArray, QTranslator, QCoreApplication, QThread,
-                         QByteArray, QTranslator, QCoreApplication, QThread, \
+                    QEvent, QTimer, pyqtSignal, QDate, QDesktopServices,
-                         QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \
+                    QFileDialog, QFileIconProvider,
-                         QFileDialog, QFileIconProvider, \
+                    QIcon, QApplication, QDialog, QUrl, QFont)
-                         QIcon, QApplication, QDialog, QUrl, QFont
 
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
 from calibre.constants import islinux, iswindows, isfreebsd, isfrozen, isosx
 from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
 from calibre.utils.localization import set_qt_translator
-from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.date import UNDEFINED_DATE
 
@@ -156,7 +154,9 @@ def _config():
     c.add_opt('plugin_search_history', default=[],
         help='Search history for the recipe scheduler')
     c.add_opt('worker_limit', default=6,
-            help=_('Maximum number of waiting worker processes'))
+            help=_(
+        'Maximum number of simultaneous conversion/news download jobs. '
+        'This number is twice the actual value for historical reasons.'))
     c.add_opt('get_social_metadata', default=True,
             help=_('Download social metadata (tags/rating/etc.)'))
     c.add_opt('overwrite_author_title_metadata', default=True,
@@ -330,6 +330,7 @@ class GetMetadata(QObject):
                   id, args, kwargs)
 
     def _from_formats(self, id, args, kwargs):
+        from calibre.ebooks.metadata.meta import metadata_from_formats
         try:
             mi = metadata_from_formats(*args, **kwargs)
         except:
@@ -337,6 +338,7 @@ class GetMetadata(QObject):
         self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi)
 
     def _get_metadata(self, id, args, kwargs):
+        from calibre.ebooks.metadata.meta import get_metadata
         try:
             mi = get_metadata(*args, **kwargs)
         except:
@@ -738,3 +740,4 @@ def build_forms(srcdir, info=None):
 _df = os.environ.get('CALIBRE_DEVELOP_FROM', None)
 if _df and os.path.exists(_df):
     build_forms(_df)
+

src/calibre/gui2/actions/annotate.py

@@ -22,7 +22,7 @@ class FetchAnnotationsAction(InterfaceAction):
     action_type = 'current'
 
     def genesis(self):
-        pass
+        self.qaction.triggered.connect(self.fetch_annotations)
 
     def fetch_annotations(self, *args):
         # Generate a path_map from selected ids
@@ -52,6 +52,10 @@ class FetchAnnotationsAction(InterfaceAction):
             return path_map
 
         device = self.gui.device_manager.device
+        if not getattr(device, 'SUPPORTS_ANNOTATIONS', False):
+            return error_dialog(self.gui, _('Not supported'),
+                    _('Fetching annotations is not supported for this device'),
+                    show=True)
 
         if self.gui.current_view() is not self.gui.library_view:
             return error_dialog(self.gui, _('Use library only'),

src/calibre/gui2/actions/edit_metadata.py

@@ -8,14 +8,15 @@ __docformat__ = 'restructuredtext en'
 import os
 from functools import partial
 
-from PyQt4.Qt import Qt, QMenu, QModelIndex
+from PyQt4.Qt import Qt, QMenu, QModelIndex, QTimer
 
-from calibre.gui2 import error_dialog, config, Dispatcher
+from calibre.gui2 import error_dialog, config, Dispatcher, question_dialog
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
+from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.icu import sort_key
 from calibre.utils.config import test_eight_code
 
@@ -78,6 +79,7 @@ class EditMetadataAction(InterfaceAction):
         self.qaction.setEnabled(enabled)
         self.action_merge.setEnabled(enabled)
 
+    # Download metadata {{{
     def download_metadata(self, ids=None):
         if ids is None:
             rows = self.gui.library_view.selectionModel().selectedRows()
@@ -88,14 +90,73 @@ class EditMetadataAction(InterfaceAction):
             ids = [db.id(row.row()) for row in rows]
         from calibre.gui2.metadata.bulk_download2 import start_download
         start_download(self.gui, ids,
-                Dispatcher(self.bulk_metadata_downloaded))
+                Dispatcher(self.metadata_downloaded))
 
-    def bulk_metadata_downloaded(self, job):
+    def metadata_downloaded(self, job):
         if job.failed:
             self.gui.job_exception(job, dialog_title=_('Failed to download metadata'))
             return
-        from calibre.gui2.metadata.bulk_download2 import proceed
+        from calibre.gui2.metadata.bulk_download2 import get_job_details
-        proceed(self.gui, job)
+        id_map, failed_ids, failed_covers, all_failed, det_msg = \
+                                            get_job_details(job)
+        if all_failed:
+            return error_dialog(self.gui, _('Download failed'),
+            _('Failed to download metadata or covers for any of the %d'
+               ' book(s).') % len(id_map), det_msg=det_msg, show=True)
+
+        self.gui.status_bar.show_message(_('Metadata download completed'), 3000)
+
+        msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. '
+            'Proceed with updating the metadata in your library?')%len(id_map)
+
+        show_copy_button = False
+        if failed_ids or failed_covers:
+            show_copy_button = True
+            msg += '<p>'+_('Could not download metadata and/or covers for %d of the books. Click'
+                    ' "Show details" to see which books.')%len(failed_ids)
+
+        payload = (id_map, failed_ids, failed_covers)
+        from calibre.gui2.dialogs.message_box import ProceedNotification
+        p = ProceedNotification(payload, job.html_details,
+                _('Download log'), _('Download complete'), msg,
+                det_msg=det_msg, show_copy_button=show_copy_button,
+                parent=self.gui)
+        p.proceed.connect(self.apply_downloaded_metadata)
+        p.show()
+
+    def apply_downloaded_metadata(self, payload):
+        id_map, failed_ids, failed_covers = payload
+        id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in
+            failed_ids])
+        if not id_map:
+            return
+
+        modified = set()
+        db = self.gui.current_db
+
+        for i, mi in id_map.iteritems():
+            lm = db.metadata_last_modified(i, index_is_id=True)
+            if lm > mi.last_modified:
+                title = db.title(i, index_is_id=True)
+                authors = db.authors(i, index_is_id=True)
+                if authors:
+                    authors = [x.replace('|', ',') for x in authors.split(',')]
+                    title += ' - ' + authors_to_string(authors)
+                modified.add(title)
+
+        if modified:
+            from calibre.utils.icu import lower
+
+            modified = sorted(modified, key=lower)
+            if not question_dialog(self.gui, _('Some books changed'), '<p>'+
+                    _('The metadata for some books in your library has'
+                        ' changed since you started the download. If you'
+                        ' proceed, some of those changes may be overwritten. '
+                        'Click "Show details" to see the list of changed books. '
+                        'Do you want to proceed?'), det_msg='\n'.join(modified)):
+                return
+
+        self.apply_metadata_changes(id_map)
 
     def download_metadata_old(self, checked, covers=True, set_metadata=True,
             set_social_metadata=None):
@@ -140,6 +201,7 @@ class EditMetadataAction(InterfaceAction):
                 x.updated, cr)
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()
+    # }}}
 
     def edit_metadata(self, checked, bulk=None):
         '''
@@ -466,4 +528,89 @@ class EditMetadataAction(InterfaceAction):
             self.gui.upload_collections(model.db, view=view, oncard=oncard)
             view.reset()
 
+    # Apply bulk metadata changes {{{
+    def apply_metadata_changes(self, id_map, title=None, msg=''):
+        '''
+        Apply the metadata changes in id_map to the database synchronously
+        id_map must be a mapping of ids to Metadata objects. Set any fields you
+        do not want updated in the Metadata object to null. An easy way to do
+        that is to create a metadata object as Metadata(_('Unknown')) and then
+        only set the fields you want changed on this object.
+        '''
+        if title is None:
+            title = _('Applying changed metadata')
+        self.apply_id_map = list(id_map.iteritems())
+        self.apply_current_idx = 0
+        self.apply_failures = []
+        self.applied_ids = []
+        self.apply_pd = None
+        if len(self.apply_id_map) > 1:
+            from calibre.gui2.dialogs.progress import ProgressDialog
+            self.apply_pd = ProgressDialog(title, msg, min=0,
+                    max=len(self.apply_id_map)-1, parent=self.gui,
+                    cancelable=False)
+            self.apply_pd.setModal(True)
+            self.apply_pd.show()
+        self.do_one_apply()
+
+
+    def do_one_apply(self):
+        if self.apply_current_idx >= len(self.apply_id_map):
+            return self.finalize_apply()
+
+        i, mi = self.apply_id_map[self.apply_current_idx]
+        db = self.gui.current_db
+        try:
+            set_title = not mi.is_null('title')
+            set_authors = not mi.is_null('authors')
+            db.set_metadata(i, mi, commit=False, set_title=set_title,
+                    set_authors=set_authors, notify=False)
+            self.applied_ids.append(i)
+        except:
+            import traceback
+            self.apply_failures.append((i, traceback.format_exc()))
+
+        try:
+            if mi.cover:
+                os.remove(mi.cover)
+        except:
+            pass
+
+        self.apply_current_idx += 1
+        if self.apply_pd is not None:
+            self.apply_pd.value += 1
+        QTimer.singleShot(50, self.do_one_apply)
+
+    def finalize_apply(self):
+        db = self.gui.current_db
+        db.commit()
+
+        if self.apply_pd is not None:
+            self.apply_pd.hide()
+
+        if self.apply_failures:
+            msg = []
+            for i, tb in self.apply_failures:
+                title = db.title(i, index_is_id=True)
+                authors = db.authors(i, index_is_id=True)
+                if authors:
+                    authors = [x.replace('|', ',') for x in authors.split(',')]
+                    title += ' - ' + authors_to_string(authors)
+                msg.append(title+'\n\n'+tb+'\n'+('*'*80))
+
+            error_dialog(self.gui, _('Some failures'),
+                _('Failed to apply updated metadata for some books'
+                    ' in your library. Click "Show Details" to see '
+                    'details.'), det_msg='\n\n'.join(msg), show=True)
+        if self.applied_ids:
+            cr = self.gui.library_view.currentIndex().row()
+            self.gui.library_view.model().refresh_ids(
+                self.applied_ids, cr)
+            if self.gui.cover_flow:
+                self.gui.cover_flow.dataChanged()
+
+        self.apply_id_map = []
+        self.apply_pd = None
+
+    # }}}
 

src/calibre/gui2/actions/store.py

@@ -11,6 +11,7 @@ from functools import partial
 from PyQt4.Qt import QMenu
 
 from calibre.gui2.actions import InterfaceAction
+from calibre.gui2.dialogs.confirm_delete import confirm
 
 class StoreAction(InterfaceAction):
 
@@ -31,9 +32,35 @@ class StoreAction(InterfaceAction):
         self.qaction.setMenu(self.store_menu)
 
     def search(self):
-        from calibre.gui2.store.search import SearchDialog
+        self.show_disclaimer()
+        from calibre.gui2.store.search.search import SearchDialog
         sd = SearchDialog(self.gui.istores, self.gui)
         sd.exec_()
 
     def open_store(self, store_plugin):
+        self.show_disclaimer()
         store_plugin.open(self.gui)
+
+    def show_disclaimer(self):
+        confirm(('<p>' +
+            _('Calibre helps you find the ebooks you want by searching '
+            'the websites of various commercial and public domain '
+            'book sources for you.') +
+            '<p>' +
+            _('Using the integrated search you can easily find which '
+            'store has the book you are looking for, at the best price. '
+            'You also get DRM status and other useful information.')
+            + '<p>' +
+            _('All transactions (paid or otherwise) are handled between '
+            'you and the book seller. '
+            'Calibre is not part of this process and any issues related '
+            'to a purchase should be directed to the website you are '
+            'buying from. Be sure to double check that any books you get '
+            'will work with your e-book reader, especially if the book you '
+            'are buying has '
+            '<a href="http://drmfree.calibre-ebook.com/about#drm">DRM</a>.'
+            )), 'about_get_books_msg',
+            parent=self.gui, show_cancel_button=False,
+            confirm_msg=_('Show this message again'),
+            pixmap='dialog_information.png', title=_('About Get Books'))
+

src/calibre/gui2/book_details.py

@@ -418,6 +418,7 @@ class BookDetails(QWidget): # {{{
             if y is None:
                 # Local image
                 self.cover_view.paste_from_clipboard(x)
+                self.update_layout()
             else:
                 self.remote_file_dropped.emit(x, y)
                 # We do not support setting cover *and* adding formats for
@@ -449,6 +450,7 @@ class BookDetails(QWidget): # {{{
         self.setAcceptDrops(True)
         self._layout = DetailsLayout(vertical, self)
         self.setLayout(self._layout)
+        self.current_path = ''
 
         self.cover_view = CoverView(vertical, self)
         self.cover_view.cover_changed.connect(self.cover_changed.emit)
@@ -482,6 +484,10 @@ class BookDetails(QWidget): # {{{
     def show_data(self, data):
         self.book_info.show_data(data)
         self.cover_view.show_data(data)
+        self.current_path = data.get(_('Path'), '')
+        self.update_layout()
+
+    def update_layout(self):
         self._layout.do_layout(self.rect())
         try:
             sz = self.cover_view.pixmap.size()
@@ -489,7 +495,7 @@ class BookDetails(QWidget): # {{{
             sz = QSize(0, 0)
         self.setToolTip(
             '<p>'+_('Double-click to open Book Details window') +
-                '<br><br>' + _('Path') + ': ' + data.get(_('Path'), '') +
+            '<br><br>' + _('Path') + ': ' + self.current_path +
             '<br><br>' + _('Cover size: %dx%d')%(sz.width(), sz.height())
         )
 

src/calibre/gui2/device.py

@@ -25,7 +25,6 @@ from calibre.devices.errors import FreeSpaceError
 from calibre.devices.apple.driver import ITUNES_ASYNC
 from calibre.devices.folder_device.driver import FOLDER_DEVICE
 from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi
-from calibre.ebooks.metadata.meta import set_metadata
 from calibre.constants import DEBUG
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import thumbnail
@@ -334,6 +333,7 @@ class DeviceManager(Thread): # {{{
 
     def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None):
         '''Upload books to device: '''
+        from calibre.ebooks.metadata.meta import set_metadata
         if hasattr(self.connected_device, 'set_plugboards') and \
                 callable(self.connected_device.set_plugboards):
             self.connected_device.set_plugboards(plugboards, find_plugboard)
@@ -591,11 +591,16 @@ class DeviceSignals(QObject):
     #: This signal is emitted once, after metadata is downloaded from the
     #: connected device.
     #: The sequence: gui.device_manager.is_device_connected will become True,
+    #: and the device_connection_changed signal will be emitted,
     #: then sometime later gui.device_metadata_available will be signaled.
     #: This does not mean that there are no more jobs running. Automatic metadata
     #: management might have kicked off a sync_booklists to write new metadata onto
     #: the device, and that job might still be running when the signal is emitted.
     device_metadata_available = pyqtSignal()
+
+    #: This signal is emitted once when the device is detected and once when
+    #: it is disconnected. If the parameter is True, then it is a connection,
+    #: otherwise a disconnection.
     device_connection_changed = pyqtSignal(object)
 
 device_signals = DeviceSignals()

src/calibre/gui2/dialogs/confirm_delete.py

@@ -24,11 +24,18 @@ class Dialog(QDialog, Ui_Dialog):
         dynamic[confirm_config_name(self.name)] = self.again.isChecked()
 
 
-def confirm(msg, name, parent=None, pixmap='dialog_warning.png'):
+def confirm(msg, name, parent=None, pixmap='dialog_warning.png', title=None,
+        show_cancel_button=True, confirm_msg=None):
     if not dynamic.get(confirm_config_name(name), True):
         return True
     d = Dialog(msg, name, parent)
     d.label.setPixmap(QPixmap(I(pixmap)))
     d.setWindowIcon(QIcon(I(pixmap)))
+    if title is not None:
+        d.setWindowTitle(title)
+    if not show_cancel_button:
+        d.buttonBox.button(d.buttonBox.Cancel).setVisible(False)
+    if confirm_msg is not None:
+        d.again.setText(confirm_msg)
     d.resize(d.sizeHint())
     return d.exec_() == d.Accepted

src/calibre/gui2/dialogs/message_box.py

@@ -6,13 +6,13 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from PyQt4.Qt import QDialog, QIcon, QApplication, QSize, QKeySequence, \
+from PyQt4.Qt import (QDialog, QIcon, QApplication, QSize, QKeySequence,
-    QAction, Qt
+    QAction, Qt, pyqtSignal, QTextBrowser, QDialogButtonBox, QVBoxLayout)
 
 from calibre.constants import __version__
 from calibre.gui2.dialogs.message_box_ui import Ui_Dialog
 
-class MessageBox(QDialog, Ui_Dialog):
+class MessageBox(QDialog, Ui_Dialog): # {{{
 
     ERROR = 0
     WARNING = 1
@@ -111,6 +111,81 @@ class MessageBox(QDialog, Ui_Dialog):
         self.det_msg_toggle.setVisible(bool(msg))
         self.det_msg.setVisible(False)
         self.do_resize()
+# }}}
+
+class ViewLog(QDialog): # {{{
+
+    def __init__(self, title, html, parent=None):
+        QDialog.__init__(self, parent)
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+
+        self.tb = QTextBrowser(self)
+        self.tb.setHtml('<pre style="font-family: monospace">%s</pre>' % html)
+        l.addWidget(self.tb)
+
+        self.bb = QDialogButtonBox(QDialogButtonBox.Ok)
+        self.bb.accepted.connect(self.accept)
+        self.bb.rejected.connect(self.reject)
+        self.copy_button = self.bb.addButton(_('Copy to clipboard'),
+                self.bb.ActionRole)
+        self.copy_button.setIcon(QIcon(I('edit-copy.png')))
+        self.copy_button.clicked.connect(self.copy_to_clipboard)
+        l.addWidget(self.bb)
+        self.setModal(False)
+        self.resize(QSize(700, 500))
+        self.setWindowTitle(title)
+        self.setWindowIcon(QIcon(I('debug.png')))
+        self.show()
+
+    def copy_to_clipboard(self):
+        txt = self.tb.toPlainText()
+        QApplication.clipboard().setText(txt)
+# }}}
+
+class ProceedNotification(MessageBox): # {{{
+
+    proceed = pyqtSignal(object)
+
+    def __init__(self, payload, html_log, log_viewer_title, title, msg, det_msg='', show_copy_button=False, parent=None):
+        '''
+        A non modal popup that notifies the user that a background task has
+        been completed. If they user clicks yes, the proceed signal is emitted
+        with payload as its argument.
+
+        :param payload: Arbitrary object, emitted in the proceed signal
+        :param html_log: An HTML or plain text log
+        :param log_viewer_title: The title for the log viewer window
+        :param title: The title fo rthis popup
+        :param msg: The msg to display
+        :param det_msg: Detailed message
+        '''
+        MessageBox.__init__(self, MessageBox.QUESTION, title, msg,
+                det_msg=det_msg, show_copy_button=show_copy_button,
+                parent=parent)
+        self.payload = payload
+        self.html_log = html_log
+        self.log_viewer_title = log_viewer_title
+        self.finished.connect(self.do_proceed)
+
+        self.vlb = self.bb.addButton(_('View log'), self.bb.ActionRole)
+        self.vlb.setIcon(QIcon(I('debug.png')))
+        self.vlb.clicked.connect(self.show_log)
+        self.det_msg_toggle.setVisible(bool(det_msg))
+        self.setModal(False)
+
+    def show_log(self):
+        self.log_viewer = ViewLog(self.log_viewer_title, self.html_log,
+                parent=self)
+
+    def do_proceed(self, result):
+        if result == self.Accepted:
+            self.proceed.emit(self.payload)
+        try:
+            self.proceed.disconnect()
+        except:
+            pass
+# }}}
 
 if __name__ == '__main__':
     app = QApplication([])

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -13,7 +13,6 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_sort
 from calibre.ebooks.metadata.book.base import composite_formatter
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, \
     gprefs, question_dialog
@@ -26,6 +25,7 @@ from calibre.utils.magick.draw import identify_data
 from calibre.utils.date import qt_to_dt
 
 def get_cover_data(path): # {{{
+    from calibre.ebooks.metadata.meta import get_metadata
     old = prefs['read_file_metadata']
     if not old:
         prefs['read_file_metadata'] = True

src/calibre/gui2/dialogs/metadata_single.py

@@ -25,7 +25,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata import string_to_authors, \
         authors_to_string, check_isbn, title_sort
 from calibre.ebooks.metadata.covers import download_cover
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp
@@ -353,6 +352,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             self.formats_changed = True
 
     def get_selected_format_metadata(self):
+        from calibre.ebooks.metadata.meta import get_metadata
         old = prefs['read_file_metadata']
         if not old:
             prefs['read_file_metadata'] = True

src/calibre/gui2/jobs.py

@@ -169,11 +169,11 @@ class JobManager(QAbstractTableModel): # {{{
                 job.update()
                 if orig_state != job.run_state:
                     needs_reset = True
+                    if job.is_finished:
+                        self.job_done.emit(len(self.unfinished_jobs()))
             if needs_reset:
                 self.jobs.sort()
                 self.reset()
-                if job.is_finished:
-                    self.job_done.emit(len(self.unfinished_jobs()))
             else:
                 for job in jobs:
                     idx = self.jobs.index(job)

src/calibre/gui2/library/models.py

@@ -18,7 +18,6 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.utils.icu import sort_key
-from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
     REGEXP_MATCH, MetadataBackup, force_to_bool
@@ -478,6 +477,7 @@ class BooksModel(QAbstractTableModel): # {{{
     def get_preferred_formats_from_ids(self, ids, formats,
                               set_metadata=False, specific_format=None,
                               exclude_auto=False, mode='r+b'):
+        from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
         ans = []
         need_auto = []
         if specific_format is not None:
@@ -526,6 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{
     def get_preferred_formats(self, rows, formats, paths=False,
                               set_metadata=False, specific_format=None,
                               exclude_auto=False):
+        from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
         ans = []
         need_auto = []
         if specific_format is not None:

src/calibre/gui2/main.py

@@ -19,6 +19,9 @@ from calibre.utils.config import prefs, dynamic
 from calibre.library.database2 import LibraryDatabase2
 from calibre.library.sqlite import sqlite, DatabaseException
 
+if iswindows:
+    winutil = plugins['winutil'][0]
+
 def option_parser():
     parser = _option_parser('''\
 %prog [opts] [path_to_ebook]
@@ -80,8 +83,7 @@ def get_library_path(parent=None):
     if library_path is None: # Need to migrate to new database layout
         base = os.path.expanduser('~')
         if iswindows:
-            base = plugins['winutil'][0].special_folder_path(
+            base = winutil.special_folder_path(winutil.CSIDL_PERSONAL)
-                    plugins['winutil'][0].CSIDL_PERSONAL)
             if not base or not os.path.exists(base):
                 from PyQt4.Qt import QDir
                 base = unicode(QDir.homePath()).replace('/', os.sep)

src/calibre/gui2/metadata/basic_widgets.py

@@ -24,7 +24,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.gui2 import file_icon_provider, UNDEFINED_QDATE, UNDEFINED_DATE, \
         choose_files, error_dialog, choose_images, question_dialog
 from calibre.utils.date import local_tz, qt_to_dt
-from calibre import strftime, fit_image
+from calibre import strftime
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.customize.ui import run_plugins_on_import
 from calibre.utils.date import utcfromtimestamp
@@ -672,12 +672,7 @@ class Cover(ImageView): # {{{
         self.frame_size = (sz.width()//3, sz.height())
 
     def sizeHint(self):
-        sz = ImageView.sizeHint(self)
+        sz = QSize(self.frame_size[0], self.frame_size[1])
-        w, h = sz.width(), sz.height()
-        resized, nw, nh = fit_image(w, h, self.frame_size[0],
-                self.frame_size[1])
-        if resized:
-            sz = QSize(nw, nh)
         return sz
 
     def select_cover(self, *args):

src/calibre/gui2/metadata/bulk_download2.py

@@ -7,19 +7,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
 from functools import partial
 from itertools import izip
+from threading import Event
 
-from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize,
+from PyQt4.Qt import (QIcon, QDialog,
-        QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar,
+        QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt)
-        QGridLayout, QPixmap, Qt)
 
-from calibre.gui2.dialogs.message_box import MessageBox
 from calibre.gui2.threaded_jobs import ThreadedJob
-from calibre.utils.icu import lower
-from calibre.ebooks.metadata import authors_to_string
-from calibre.gui2 import question_dialog, error_dialog
 from calibre.ebooks.metadata.sources.identify import identify, msprefs
 from calibre.ebooks.metadata.sources.covers import download_cover
 from calibre.ebooks.metadata.book.base import Metadata
@@ -107,183 +102,8 @@ def start_download(gui, ids, callback):
     gui.status_bar.show_message(_('Metadata download started'), 3000)
 # }}}
 
-class ViewLog(QDialog): # {{{
+def get_job_details(job):
-
+    id_map, failed_ids, failed_covers, title_map, all_failed = job.result
-    def __init__(self, html, parent=None):
-        QDialog.__init__(self, parent)
-        self.l = l = QVBoxLayout()
-        self.setLayout(l)
-
-        self.tb = QTextBrowser(self)
-        self.tb.setHtml('<pre style="font-family: monospace">%s</pre>' % html)
-        l.addWidget(self.tb)
-
-        self.bb = QDialogButtonBox(QDialogButtonBox.Ok)
-        self.bb.accepted.connect(self.accept)
-        self.bb.rejected.connect(self.reject)
-        self.copy_button = self.bb.addButton(_('Copy to clipboard'),
-                self.bb.ActionRole)
-        self.copy_button.setIcon(QIcon(I('edit-copy.png')))
-        self.copy_button.clicked.connect(self.copy_to_clipboard)
-        l.addWidget(self.bb)
-        self.setModal(False)
-        self.resize(QSize(700, 500))
-        self.setWindowTitle(_('Download log'))
-        self.setWindowIcon(QIcon(I('debug.png')))
-        self.show()
-
-    def copy_to_clipboard(self):
-        txt = self.tb.toPlainText()
-        QApplication.clipboard().setText(txt)
-
-_vl = None
-def view_log(job, parent):
-    global _vl
-    _vl = ViewLog(job.html_details, parent)
-
-# }}}
-
-# Apply downloaded metadata {{{
-class ApplyDialog(QDialog):
-
-    def __init__(self, gui):
-        QDialog.__init__(self, gui)
-
-        self.l = l = QVBoxLayout()
-        self.setLayout(l)
-        l.addWidget(QLabel(_('Applying downloaded metadata to your library')))
-
-        self.pb = QProgressBar(self)
-        l.addWidget(self.pb)
-
-        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
-        self.bb.rejected.connect(self.reject)
-        l.addWidget(self.bb)
-
-        self.gui = gui
-        self.timer = QTimer(self)
-        self.timer.timeout.connect(self.do_one)
-
-    def start(self, id_map):
-        self.id_map = list(id_map.iteritems())
-        self.current_idx = 0
-        self.failures = []
-        self.ids = []
-        self.canceled = False
-        self.pb.setMinimum(0)
-        self.pb.setMaximum(len(id_map))
-        self.timer.start(50)
-
-    def do_one(self):
-        if self.canceled:
-            return
-        if self.current_idx >= len(self.id_map):
-            self.timer.stop()
-            self.finalize()
-            return
-
-        i, mi = self.id_map[self.current_idx]
-        db = self.gui.current_db
-        try:
-            set_title = not mi.is_null('title')
-            set_authors = not mi.is_null('authors')
-            db.set_metadata(i, mi, commit=False, set_title=set_title,
-                    set_authors=set_authors)
-            self.ids.append(i)
-        except:
-            import traceback
-            self.failures.append((i, traceback.format_exc()))
-
-        try:
-            if mi.cover:
-                os.remove(mi.cover)
-        except:
-            pass
-
-        self.pb.setValue(self.pb.value()+1)
-        self.current_idx += 1
-
-    def reject(self):
-        self.canceled = True
-        self.timer.stop()
-        QDialog.reject(self)
-
-    def finalize(self):
-        if self.canceled:
-            return
-        if self.failures:
-            msg = []
-            db = self.gui.current_db
-            for i, tb in self.failures:
-                title = db.title(i, index_is_id=True)
-                authors = db.authors(i, index_is_id=True)
-                if authors:
-                    authors = [x.replace('|', ',') for x in authors.split(',')]
-                    title += ' - ' + authors_to_string(authors)
-                msg.append(title+'\n\n'+tb+'\n'+('*'*80))
-
-            parent = self if self.isVisible() else self.parent()
-            error_dialog(parent, _('Some failures'),
-                _('Failed to apply updated metadata for some books'
-                    ' in your library. Click "Show Details" to see '
-                    'details.'), det_msg='\n\n'.join(msg), show=True)
-        if self.ids:
-            cr = self.gui.library_view.currentIndex().row()
-            self.gui.library_view.model().refresh_ids(
-                self.ids, cr)
-            if self.gui.cover_flow:
-                self.gui.cover_flow.dataChanged()
-        self.accept()
-
-_amd = None
-def apply_metadata(job, gui, q, result):
-    global _amd
-    q.vlb.clicked.disconnect()
-    q.finished.disconnect()
-    if result != q.Accepted:
-        return
-    id_map, failed_ids, failed_covers, title_map = job.result
-    id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in
-        failed_ids])
-    if not id_map:
-        return
-
-    modified = set()
-    db = gui.current_db
-
-    for i, mi in id_map.iteritems():
-        lm = db.metadata_last_modified(i, index_is_id=True)
-        if lm > mi.last_modified:
-            title = db.title(i, index_is_id=True)
-            authors = db.authors(i, index_is_id=True)
-            if authors:
-                authors = [x.replace('|', ',') for x in authors.split(',')]
-                title += ' - ' + authors_to_string(authors)
-            modified.add(title)
-
-    if modified:
-        modified = sorted(modified, key=lower)
-        if not question_dialog(gui, _('Some books changed'), '<p>'+
-                _('The metadata for some books in your library has'
-                    ' changed since you started the download. If you'
-                    ' proceed, some of those changes may be overwritten. '
-                    'Click "Show details" to see the list of changed books. '
-                    'Do you want to proceed?'), det_msg='\n'.join(modified)):
-            return
-
-    if _amd is None:
-        _amd = ApplyDialog(gui)
-    _amd.start(id_map)
-    if len(id_map) > 3:
-        _amd.exec_()
-
-def proceed(gui, job):
-    gui.status_bar.show_message(_('Metadata download completed'), 3000)
-    id_map, failed_ids, failed_covers, title_map = job.result
-    fmsg = det_msg = ''
-    if failed_ids or failed_covers:
-        fmsg = '<p>'+_('Could not download metadata and/or covers for %d of the books. Click'
-                ' "Show details" to see which books.')%len(failed_ids)
     det_msg = []
     for i in failed_ids | failed_covers:
         title = title_map[i]
@@ -292,24 +112,13 @@ def proceed(gui, job):
         if i in failed_covers:
             title += (' ' + _('(Failed cover)'))
         det_msg.append(title)
-    msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. '
+    det_msg = '\n'.join(det_msg)
-        'Proceed with updating the metadata in your library?')%len(id_map)
+    return id_map, failed_ids, failed_covers, all_failed, det_msg
-    q = MessageBox(MessageBox.QUESTION, _('Download complete'),
-            msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids),
-            parent=gui)
-    q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole)
-    q.vlb.setIcon(QIcon(I('debug.png')))
-    q.vlb.clicked.connect(partial(view_log, job, q))
-    q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers))
-    q.setModal(False)
-    q.show()
-    q.finished.connect(partial(apply_metadata, job, gui, q))
-
-# }}}
 
 def merge_result(oldmi, newmi):
     dummy = Metadata(_('Unknown'))
     for f in msprefs['ignore_fields']:
+        if ':' not in f:
             setattr(newmi, f, getattr(dummy, f))
     fields = set()
     for plugin in metadata_plugins(['identify']):
@@ -335,6 +144,11 @@ def download(ids, db, do_identify, covers,
     title_map = {}
     ans = {}
     count = 0
+    all_failed = True
+    '''
+    # Test apply dialog
+    all_failed = do_identify = covers = False
+    '''
     for i, mi in izip(ids, metadata):
         if abort.is_set():
             log.error('Aborting...')
@@ -344,11 +158,12 @@ def download(ids, db, do_identify, covers,
         if do_identify:
             results = []
             try:
-                results = identify(log, abort, title=title, authors=authors,
+                results = identify(log, Event(), title=title, authors=authors,
                     identifiers=identifiers)
             except:
                 pass
             if results:
+                all_failed = False
                 mi = merge_result(mi, results[0])
                 identifiers = mi.identifiers
                 if not mi.is_null('rating'):
@@ -366,6 +181,7 @@ def download(ids, db, do_identify, covers,
                 with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
                     f.write(cdata[-1])
                     mi.cover = f.name
+                all_failed = False
             else:
                 failed_covers.add(i)
         ans[i] = mi
@@ -373,7 +189,7 @@ def download(ids, db, do_identify, covers,
         notifications.put((count/len(ids),
             _('Downloaded %d of %d')%(count, len(ids))))
     log('Download complete, with %d failures'%len(failed_ids))
-    return (ans, failed_ids, failed_covers, title_map)
+    return (ans, failed_ids, failed_covers, title_map, all_failed)
 
 
 

src/calibre/gui2/metadata/single.py

@@ -326,6 +326,7 @@ class MetadataSingleDialogBase(ResizableDialog):
             mi = d.book
             dummy = Metadata(_('Unknown'))
             for f in msprefs['ignore_fields']:
+                if ':' not in f:
                     setattr(mi, f, getattr(dummy, f))
             if mi is not None:
                 self.update_from_mi(mi)

src/calibre/gui2/metadata/single_download.py

@@ -15,10 +15,10 @@ from operator import attrgetter
 from Queue import Queue, Empty
 
 from PyQt4.Qt import (QStyledItemDelegate, QTextDocument, QRectF, QIcon, Qt,
-        QStyle, QApplication, QDialog, QVBoxLayout, QLabel, QDialogButtonBox,
+        QApplication, QDialog, QVBoxLayout, QLabel, QDialogButtonBox,
         QStackedWidget, QWidget, QTableView, QGridLayout, QFontInfo, QPalette,
         QTimer, pyqtSignal, QAbstractTableModel, QVariant, QSize, QListView,
-        QPixmap, QAbstractListModel, QColor, QRect, QTextBrowser)
+        QPixmap, QAbstractListModel, QColor, QRect, QTextBrowser, QModelIndex)
 from PyQt4.QtWebKit import QWebView
 
 from calibre.customize.ui import metadata_plugins
@@ -52,12 +52,9 @@ class RichTextDelegate(QStyledItemDelegate): # {{{
         return ans
 
     def paint(self, painter, option, index):
+        QStyledItemDelegate.paint(self, painter, option, QModelIndex())
         painter.save()
         painter.setClipRect(QRectF(option.rect))
-        if hasattr(QStyle, 'CE_ItemViewItem'):
-            QApplication.style().drawControl(QStyle.CE_ItemViewItem, option, painter)
-        elif option.state & QStyle.State_Selected:
-            painter.fillRect(option.rect, option.palette.highlight())
         painter.translate(option.rect.topLeft())
         self.to_doc(index).drawContents(painter)
         painter.restore()
@@ -116,14 +113,17 @@ class CoverDelegate(QStyledItemDelegate): # {{{
 
     def paint(self, painter, option, index):
         QStyledItemDelegate.paint(self, painter, option, index)
-        # Ensure the cover is rendered over any selection rect
         style = QApplication.style()
-        style.drawItemPixmap(painter, option.rect, Qt.AlignTop|Qt.AlignHCenter,
+        waiting = self.timer.isActive() and index.data(Qt.UserRole).toBool()
-            QPixmap(index.data(Qt.DecorationRole)))
+        if waiting:
-        if self.timer.isActive() and index.data(Qt.UserRole).toBool():
             rect = QRect(0, 0, self.spinner_width, self.spinner_width)
             rect.moveCenter(option.rect.center())
             self.draw_spinner(painter, rect)
+        else:
+            # Ensure the cover is rendered over any selection rect
+            style.drawItemPixmap(painter, option.rect, Qt.AlignTop|Qt.AlignHCenter,
+                QPixmap(index.data(Qt.DecorationRole)))
+
 # }}}
 
 class ResultsModel(QAbstractTableModel): # {{{
@@ -949,7 +949,7 @@ class CoverFetch(QDialog): # {{{
 # }}}
 
 if __name__ == '__main__':
-    #DEBUG_DIALOG = True
+    DEBUG_DIALOG = True
     app = QApplication([])
     d = FullFetch()
     d.start(title='great gatsby', authors=['fitzgerald'])

src/calibre/gui2/preferences/metadata_sources.py

@@ -259,6 +259,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('wait_after_first_identify_result', msprefs)
         r('wait_after_first_cover_result', msprefs)
         r('swap_author_names', msprefs)
+        r('fewer_tags', msprefs)
 
         self.configure_plugin_button.clicked.connect(self.configure_plugin)
         self.sources_model = SourcesModel(self)

src/calibre/gui2/preferences/metadata_sources.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>781</width>
-    <height>300</height>
+    <height>394</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -21,7 +21,7 @@
     <widget class="QStackedWidget" name="stack">
      <widget class="QWidget" name="page">
       <layout class="QGridLayout" name="gridLayout">
-       <item row="0" column="0" rowspan="6">
+       <item row="0" column="0" rowspan="7">
         <widget class="QGroupBox" name="groupBox">
          <property name="title">
           <string>Metadata sources</string>
@@ -105,7 +105,7 @@
          </property>
         </widget>
        </item>
-       <item row="3" column="1">
+       <item row="4" column="1">
         <widget class="QLabel" name="label_2">
          <property name="text">
           <string>Max. number of &amp;tags to download:</string>
@@ -115,10 +115,10 @@
          </property>
         </widget>
        </item>
-       <item row="3" column="2">
+       <item row="4" column="2">
         <widget class="QSpinBox" name="opt_max_tags"/>
        </item>
-       <item row="4" column="1">
+       <item row="5" column="1">
         <widget class="QLabel" name="label_3">
          <property name="text">
           <string>Max. &amp;time to wait after first match is found:</string>
@@ -128,14 +128,14 @@
          </property>
         </widget>
        </item>
-       <item row="4" column="2">
+       <item row="5" column="2">
         <widget class="QSpinBox" name="opt_wait_after_first_identify_result">
          <property name="suffix">
           <string> secs</string>
          </property>
         </widget>
        </item>
-       <item row="5" column="1">
+       <item row="6" column="1">
         <widget class="QLabel" name="label_4">
          <property name="text">
           <string>Max. time to wait after first &amp;cover is found:</string>
@@ -145,13 +145,24 @@
          </property>
         </widget>
        </item>
-       <item row="5" column="2">
+       <item row="6" column="2">
         <widget class="QSpinBox" name="opt_wait_after_first_cover_result">
          <property name="suffix">
           <string> secs</string>
          </property>
         </widget>
        </item>
+       <item row="3" column="1" colspan="2">
+        <widget class="QCheckBox" name="opt_fewer_tags">
+         <property name="toolTip">
+          <string>&lt;p&gt;Different metadata sources have different sets of tags for the same book. If this option is checked, then calibre will use the smaller tag sets. These tend to be more like genres, while the larger tag sets tend to describe the books content.
+&lt;p&gt;Note that this option will only make a practical difference if one of the metadata sources has a genre like tag set for the book you are searching for. Most often, they all have large tag sets.</string>
+         </property>
+         <property name="text">
+          <string>Prefer &amp;fewer tags</string>
+         </property>
+        </widget>
+       </item>
       </layout>
      </widget>
      <widget class="QWidget" name="page_2"/>

src/calibre/gui2/preferences/misc.py

@@ -6,19 +6,27 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 
-from calibre.gui2.preferences import ConfigWidgetBase, test_widget
+from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.misc_ui import Ui_Form
 from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
 from calibre.constants import isosx
 
-# Check Integrity {{{
+class WorkersSetting(Setting):
+
+    def set_gui_val(self, val):
+        val = val//2
+        Setting.set_gui_val(self, val)
+
+    def get_gui_val(self):
+        val = Setting.get_gui_val(self)
+        return val * 2
 
 class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
     def genesis(self, gui):
         self.gui = gui
         r = self.register
-        r('worker_limit', config, restart_required=True)
+        r('worker_limit', config, restart_required=True, setting=WorkersSetting)
         r('enforce_cpu_limit', config, restart_required=True)
         self.device_detection_button.clicked.connect(self.debug_device_detection)
         self.button_open_config_dir.clicked.connect(self.open_config_dir)

src/calibre/gui2/preferences/misc.ui

@@ -17,7 +17,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label_5">
      <property name="text">
-      <string>&amp;Maximum number of waiting worker processes (needs restart):</string>
+      <string>Max. simultaneous conversion/news download jobs:</string>
      </property>
      <property name="buddy">
       <cstring>opt_worker_limit</cstring>
@@ -27,13 +27,7 @@
    <item row="0" column="1">
     <widget class="QSpinBox" name="opt_worker_limit">
      <property name="minimum">
-      <number>2</number>
+      <number>1</number>
-     </property>
-     <property name="maximum">
-      <number>10000</number>
-     </property>
-     <property name="singleStep">
-      <number>2</number>
      </property>
     </widget>
    </item>

src/calibre/gui2/preferences/plugins.py

@@ -13,9 +13,9 @@ from PyQt4.Qt import Qt, QModelIndex, QAbstractItemModel, QVariant, QIcon, \
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.plugins_ui import Ui_Form
-from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \
+from calibre.customize.ui import (initialized_plugins, is_disabled, enable_plugin,
-                                 disable_plugin, plugin_customization, add_plugin, \
+                                 disable_plugin, plugin_customization, add_plugin,
-                                 remove_plugin
+                                 remove_plugin, NameConflict)
 from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files, \
         question_dialog, gprefs
 from calibre.utils.search_query_parser import SearchQueryParser
@@ -279,7 +279,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                         ' Are you sure you want to proceed?'),
                     show_copy_button=False):
                 return
+            try:
                 plugin = add_plugin(path)
+            except NameConflict as e:
+                return error_dialog(self, _('Already exists'),
+                        unicode(e), show=True)
             self._plugin_model.populate()
             self._plugin_model.reset()
             self.changed_signal.emit()

src/calibre/gui2/store/__init__.py

@@ -46,9 +46,12 @@ class StorePlugin(object): # {{{
     '''
 
     def __init__(self, gui, name):
+        from calibre.gui2 import JSONConfig
+        
         self.gui = gui
         self.name = name
         self.base_plugin = None
+        self.config = JSONConfig('store/stores/' + self.name)
 
     def open(self, gui, parent=None, detail_item=None, external=False):
         '''
@@ -76,11 +79,17 @@ class StorePlugin(object): # {{{
         return items as a generator.
 
         Don't be lazy with the search! Load as much data as possible in the
-        :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse
+        :class:`calibre.gui2.store.search_result.SearchResult` object. 
-        multiple pages to get all of the data then do so. However, if data (such as cover_url)
+        However, if data (such as cover_url)
         isn't available because the store does not display cover images then it's okay to
         ignore it.
         
+        At the very least a :class:`calibre.gui2.store.search_result.SearchResult`
+        returned by this function must have the title, author and id.
+        
+        If you have to parse multiple pages to get all of the data then implement
+        :meth:`get_deatils` for retrieving additional information.
+
         Also, by default search results can only include ebooks. A plugin can offer users
         an option to include physical books in the search results but this must be
         disabled by default.
@@ -90,13 +99,34 @@ class StorePlugin(object): # {{{
 
         :param query: The string query search with.
         :param max_results: The maximum number of results to return.
-        :param timeout: The maximum amount of time in seconds to spend download the search results.
+        :param timeout: The maximum amount of time in seconds to spend downloading data for search results.
 
         :return: :class:`calibre.gui2.store.search_result.SearchResult` objects
         item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store.
         '''
         raise NotImplementedError()
     
+    def get_details(self, search_result, timeout=60):
+        '''
+        Delayed search for information about specific search items.
+        
+        Typically, this will be used when certain information such as
+        formats, drm status, cover url are not part of the main search
+        results and the information is on another web page.
+        
+        Using this function allows for the main information (title, author)
+        to be displayed in the search results while other information can
+        take extra time to load. Splitting retrieving data that takes longer
+        to load into a separate function will give the illusion of the search
+        being faster.
+        
+        :param search_result: A search result that need details set.
+        :param timeout: The maximum amount of time in seconds to spend downloading details.
+        
+        :return: True if the search_result was modified otherwise False
+        '''
+        return False
+
     def get_settings(self):
         '''
         This is only useful for plugins that implement

src/calibre/gui2/store/amazon_plugin.py

@@ -154,6 +154,13 @@ class AmazonKindleStore(StorePlugin):
                     cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href)
                     if cover_img:
                         cover_url = cover_img[0]
+                        parts = cover_url.split('/')
+                        bn = parts[-1]
+                        f, _, ext = bn.rpartition('.')
+                        if '_' in f:
+                            bn = f.partition('_')[0]+'_SL160_.'+ext
+                            parts[-1] = bn
+                            cover_url = '/'.join(parts)
 
                 title = ''.join(data.xpath('div[@class="productTitle"]/a/text()'))
                 author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
@@ -168,5 +175,23 @@ class AmazonKindleStore(StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = asin.strip()
+                s.formats = 'Kindle'
 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://amazon.com/dp/'
+
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'):
+                if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'):
+                    search_result.drm = SearchResult.DRM_UNLOCKED
+                else:
+                    search_result.drm = SearchResult.DRM_UNKNOWN
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True
+
+

src/calibre/gui2/store/baen_webscription_plugin.py

@@ -85,5 +85,7 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price
                 s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formats = 'RB, MOBI, EPUB, LIT, LRF, RTF, HTML'
                 
                 yield s

src/calibre/gui2/store/basic_config.py

@@ -8,14 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 from PyQt4.Qt import QWidget
 
-from calibre.gui2 import gprefs
 from calibre.gui2.store.basic_config_widget_ui import Ui_Form
 
-def save_settings(config_widget):
-    gprefs[config_widget.store.name + '_open_external'] = config_widget.open_external.isChecked()
-    tags = unicode(config_widget.tags.text())
-    gprefs[config_widget.store.name + '_tags'] = tags
-
 class BasicStoreConfigWidget(QWidget, Ui_Form):
     
     def __init__(self, store):
@@ -27,10 +21,10 @@ class BasicStoreConfigWidget(QWidget, Ui_Form):
         self.load_setings()
 
     def load_setings(self):
-        settings = self.store.get_settings()
+        config = self.store.config
         
-        self.open_external.setChecked(settings.get(self.store.name + '_open_external'))
+        self.open_external.setChecked(config.get('open_external', False))
-        self.tags.setText(settings.get(self.store.name + '_tags', ''))
+        self.tags.setText(config.get('tags', ''))
 
 class BasicStoreConfig(object):
     
@@ -41,12 +35,6 @@ class BasicStoreConfig(object):
         return BasicStoreConfigWidget(self)
 
     def save_settings(self, config_widget):
-        save_settings(config_widget)
+        self.config['open_external'] = config_widget.open_external.isChecked()
-
+        tags = unicode(config_widget.tags.text())
-    def get_settings(self):
+        self.config['tags'] = tags
-        settings = {}
-        
-        settings[self.name + '_open_external'] = gprefs.get(self.name + '_open_external', False)
-        settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
-        
-        return settings

src/calibre/gui2/store/bewrite_plugin.py

@@ -23,10 +23,9 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class BeWriteStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://www.bewrite.net/mm5/merchant.mvc?Screen=SFNT'
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             if detail_item:
                 url = url + detail_item
             open_url(QUrl(url_slash_cleaner(url)))
@@ -36,7 +35,7 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
                 detail_url = url + detail_item
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -60,14 +59,6 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
                 cover_url = ''
                 price = ''
 
-                with closing(br.open(id.strip(), timeout=timeout/4)) as nf:
-                    idata = html.fromstring(nf.read())
-                    price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()'))
-                    price = '$' + price.split('$')[-1]
-                    cover_img = idata.xpath('//div[@id="content"]//img[1]/@src')
-                    if cover_img:
-                        cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0]
-
                 counter -= 1
 
                 s = SearchResult()
@@ -76,5 +67,36 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        br = browser()
+
+        with closing(br.open(search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            
+            price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()'))
+            if not price:
+                price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "MOBI")]/text()'))
+            if not price:
+                price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "PDF")]/text()'))
+            price = '$' + price.split('$')[-1]
+            search_result.price = price.strip()
+            
+            cover_img = idata.xpath('//div[@id="content"]//img[1]/@src')
+            if cover_img:
+                cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0]
+                search_result.cover_url = cover_url.strip()
+            
+            formats = set([])
+            if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "ePub")])'):
+                formats.add('EPUB')
+            if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "PDF")])'):
+                formats.add('PDF')
+            if idata.xpath('boolean(//div[@id="content"]//td[contains(text(), "MOBI")])'):
+                formats.add('MOBI')
+            search_result.formats = ', '.join(list(formats))
+
+        return True

src/calibre/gui2/store/bn_plugin.py

@@ -25,8 +25,6 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class BNStore(BasicStoreConfig, StorePlugin):
     
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
-
         pub_id = '21000000000352219'
         # Use Kovid's affiliate id 30% of the time.
         if random.randint(1, 10) in (1, 2, 3):
@@ -40,12 +38,12 @@ class BNStore(BasicStoreConfig, StorePlugin):
                 isbn = mo.group('isbn')
                 detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=' + pub_id
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_item)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -78,5 +76,7 @@ class BNStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price
                 s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNKNOWN
+                s.formats = 'Nook'
                 
                 yield s

src/calibre/gui2/store/diesel_ebooks_plugin.py

@@ -24,7 +24,6 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class DieselEbooksStore(BasicStoreConfig, StorePlugin):
         
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://www.diesel-ebooks.com/'
 
         aff_id = '?aid=2049'
@@ -37,12 +36,12 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin):
             detail_url = url + detail_item + aff_id
         url = url + aff_id
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -75,6 +74,8 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin):
                 if price_elem:
                     price = price_elem[0]
 
+                formats = ', '.join(data.xpath('.//td[@class="format"]/text()'))
+
                 counter -= 1
                 
                 s = SearchResult()
@@ -83,5 +84,18 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = '/item/' + id.strip()
+                s.formats = formats
                 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://www.diesel-ebooks.com/item/'
+        
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'):
+                search_result.drm = SearchResult.DRM_UNLOCKED
+            else:
+                search_result.drm = SearchResult.DRM_LOCKED
+        return True

src/calibre/gui2/store/ebooks_com_plugin.py

@@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import random
+import re
 import urllib2
 from contextlib import closing
 
@@ -24,8 +25,6 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class EbookscomStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
-
         m_url = 'http://www.dpbolvw.net/'
         h_click = 'click-4879827-10364500'
         d_click = 'click-4879827-10281551'
@@ -39,12 +38,12 @@ class EbookscomStore(BasicStoreConfig, StorePlugin):
         if detail_item:
             detail_url = m_url + d_click + detail_item
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -64,15 +63,6 @@ class EbookscomStore(BasicStoreConfig, StorePlugin):
                 if not id:
                     continue
                 
-                price = ''
-                with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp:
-                    pdoc = html.fromstring(fp.read())
-                    pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()')
-                    if len(pdata) >= 2:
-                        price = pdata[1]
-                if not price:
-                    continue
-                
                 cover_url = ''.join(data.xpath('.//img[1]/@src'))
                 
                 title = ''
@@ -89,7 +79,40 @@ class EbookscomStore(BasicStoreConfig, StorePlugin):
                 s.cover_url = cover_url
                 s.title = title.strip()
                 s.author = author.strip()
-                s.price = price.strip()
                 s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip()
                 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://www.ebooks.com/ebooks/book_display.asp?IID='
+        
+        mo = re.search(r'\?IID=(?P<id>\d+)', search_result.detail_item)
+        if mo:
+            id = mo.group('id')
+        if not id:
+            return
+
+        price = _('Not Available')
+        br = browser()
+        with closing(br.open(url + id, timeout=timeout)) as nf:
+            pdoc = html.fromstring(nf.read())
+            
+            pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()')
+            if len(pdata) >= 2:
+                price = pdata[1]
+            
+            search_result.drm = SearchResult.DRM_UNLOCKED
+            for sec in ('Printing', 'Copying', 'Lending'):
+                if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec):
+                    search_result.drm = SearchResult.DRM_LOCKED
+                    break
+            
+            fdata = ', '.join(pdoc.xpath('//table[@class="price"]//tr//td[1]/text()'))
+            fdata = fdata.replace(':', '')
+            fdata = re.sub(r'\s{2,}', ' ', fdata)
+            fdata = fdata.replace(' ,', ',')
+            fdata = fdata.strip()
+            search_result.formats = fdata
+        
+        search_result.price = price.strip()
+        return True

src/calibre/gui2/store/eharlequin_plugin.py

@@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
 import random
+import re
 import urllib2
 from contextlib import closing
 
@@ -24,8 +25,6 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class EHarlequinStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
-
         m_url = 'http://www.dpbolvw.net/'
         h_click = 'click-4879827-534091'
         d_click = 'click-4879827-10375439'
@@ -39,12 +38,12 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin):
         if detail_item:
             detail_url = m_url + d_click + detail_item
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -76,5 +75,28 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip()
+                s.formats = 'EPUB'
                 
                 yield s
+                
+    def get_details(self, search_result, timeout):
+        url = 'http://ebooks.eharlequin.com/en/ContentDetails.htm?ID='
+        
+        mo = re.search(r'\?ID=(?P<id>.+)', search_result.detail_item)
+        if mo:
+            id = mo.group('id')
+        if not id:
+            return
+        
+        
+        br = browser()
+        with closing(br.open(url + id, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            drm = SearchResult.DRM_UNKNOWN
+            if idata.xpath('boolean(//div[@class="drm_head"])'):
+                if idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])'):
+                    drm = SearchResult.DRM_LOCKED
+                else:
+                    drm = SearchResult.DRM_UNLOCKED
+        search_result.drm = drm
+        return True

src/calibre/gui2/store/feedbooks_plugin.py

@@ -23,11 +23,10 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class FeedbooksStore(BasicStoreConfig, StorePlugin):
     
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://m.feedbooks.com/'
         ext_url = 'http://feedbooks.com/'
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             if detail_item:
                 ext_url = ext_url + detail_item
             open_url(QUrl(url_slash_cleaner(ext_url)))
@@ -37,7 +36,7 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin):
                 detail_url = url + detail_item
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -72,8 +71,10 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin):
                 title = ''.join(data.xpath('//h5//a/text()'))
                 author = ''.join(data.xpath('//h6//a/text()'))
                 price = ''.join(data.xpath('//a[@class="buy"]/text()'))
+                formats = 'EPUB'
                 if not price:
                     price = '$0.00'
+                    formats = 'EPUB, MOBI, PDF'
                 cover_url = ''
                 cover_url_img =  data.xpath('//img')
                 if cover_url_img:
@@ -88,5 +89,18 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.replace(' ', '').strip()
                 s.detail_item = id.strip()
+                s.formats = formats
                 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://m.feedbooks.com/'
+        
+        br = browser()
+        with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            if idata.xpath('boolean(//div[contains(@class, "m-description-long")]//p[contains(., "DRM") or contains(b, "Protection")])'):
+                search_result.drm = SearchResult.DRM_LOCKED
+            else:
+                search_result.drm = SearchResult.DRM_UNLOCKED
+        return True

src/calibre/gui2/store/gutenberg_plugin.py

@@ -23,11 +23,10 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class GutenbergStore(BasicStoreConfig, StorePlugin):
         
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://m.gutenberg.org/'
         ext_url = 'http://gutenberg.org/'
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             if detail_item:
                 ext_url = ext_url + detail_item
             open_url(QUrl(url_slash_cleaner(ext_url)))
@@ -37,7 +36,7 @@ class GutenbergStore(BasicStoreConfig, StorePlugin):
                 detail_url = url + detail_item
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -79,5 +78,15 @@ class GutenbergStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = '/ebooks/' + id.strip()
+                s.drm = SearchResult.DRM_UNLOCKED
                 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://m.gutenberg.org/'
+        
+        br = browser()
+        with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            search_result.formats = ', '.join(idata.xpath('//a[@type!="application/atom+xml"]//span[@class="title"]/text()'))
+        return True

src/calibre/gui2/store/kobo_plugin.py

@@ -24,8 +24,6 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class KoboStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
-
         m_url = 'http://www.dpbolvw.net/'
         h_click = 'click-4879827-10762497'
         d_click = 'click-4879827-10772898'
@@ -39,12 +37,12 @@ class KoboStore(BasicStoreConfig, StorePlugin):
         if detail_item:
             detail_url = m_url + d_click + detail_item
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -63,7 +61,7 @@ class KoboStore(BasicStoreConfig, StorePlugin):
                 if not id:
                     continue
 
-                price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()'))
+                price = ''.join(data.xpath('.//li[@class="OurPrice"]/strong/text()'))
                 if not price:
                     price = '$0.00'
                 
@@ -71,6 +69,7 @@ class KoboStore(BasicStoreConfig, StorePlugin):
                 
                 title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()'))
                 author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()'))
+                drm = data.xpath('boolean(.//span[@class="SCAvailibilityFormatsText" and contains(text(), "DRM")])')
 
                 counter -= 1
                 
@@ -80,5 +79,7 @@ class KoboStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = '?url=http://www.kobobooks.com/' + id.strip()
+                s.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED
+                s.formats = 'EPUB'
                 
                 yield s

src/calibre/gui2/store/manybooks_plugin.py

@@ -24,19 +24,18 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class ManyBooksStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://manybooks.net/'
 
         detail_url = None
         if detail_item:
             detail_url = url + detail_item
         
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
         else:
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -89,5 +88,7 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price.strip()
                 s.detail_item = '/titles/' + id
+                s.drm = SearchResult.DRM_UNLOCKED
+                s.formts = 'EPUB, PDB (eReader, PalmDoc, zTXT, Plucker, iSilo), FB2, ZIP, AZW, MOBI, PRC, LIT, PKG, PDF, TXT, RB, RTF, LRF, TCR, JAR'
                 
                 yield s

src/calibre/gui2/store/mobileread_plugin.py

@@ -24,26 +24,23 @@ from calibre.gui2.store.basic_config import BasicStoreConfig
 from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog
 from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
-from calibre.utils.config import DynamicConfig
 from calibre.utils.icu import sort_key
 
 class MobileReadStore(BasicStoreConfig, StorePlugin):
 
     def genesis(self):
-        self.config = DynamicConfig('store_' + self.name)
         self.rlock = RLock()
 
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://www.mobileread.com/'
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             open_url(QUrl(detail_item if detail_item else url))
         else:
             if detail_item:
                 d = WebStoreDialog(self.gui, url, parent, detail_item)
                 d.setWindowTitle(self.name)
-                d.set_tags(settings.get(self.name + '_tags', ''))
+                d.set_tags(self.config.get('tags', ''))
                 d.exec_()
             else:
                 d = MobeReadStoreDialog(self, parent)
@@ -76,13 +73,14 @@ class MobileReadStore(BasicStoreConfig, StorePlugin):
         matches = heapq.nlargest(max_results, matches)
         for score, book in matches:
             book.price = '$0.00'
+            book.drm = SearchResult.DRM_UNLOCKED
             yield book
 
     def update_book_list(self, timeout=10):
         with self.rlock:
             url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
 
-            last_download = self.config.get(self.name + '_last_download', None)
+            last_download = self.config.get('last_download', None)
             # Don't update the book list if our cache is less than one week old.
             if last_download and (time.time() - last_download) < 604800:
                 return
@@ -96,15 +94,15 @@ class MobileReadStore(BasicStoreConfig, StorePlugin):
             if not raw_data:
                 return
 
-            # Turn books listed in the HTML file into BookRef's.
+            # Turn books listed in the HTML file into SearchResults's.
             books = []
             try:
                 data = html.fromstring(raw_data)
                 for book_data in data.xpath('//ul/li'):
-                    book = BookRef()
+                    book = SearchResult()
                     book.detail_item = ''.join(book_data.xpath('.//a/@href'))
-                    book.format = ''.join(book_data.xpath('.//i/text()'))
+                    book.formats = ''.join(book_data.xpath('.//i/text()'))
-                    book.format = book.format.strip()
+                    book.formats = book.formats.strip()
 
                     text = ''.join(book_data.xpath('.//a/text()'))
                     if ':' in text:
@@ -117,20 +115,34 @@ class MobileReadStore(BasicStoreConfig, StorePlugin):
 
             # Save the book list and it's create time.
             if books:
-                self.config[self.name + '_last_download'] = time.time()
+                self.config['last_download'] = time.time()
-                self.config[self.name + '_book_list'] = books
+                self.config['book_list'] = self.seralize_books(books)
 
     def get_book_list(self, timeout=10):
         self.update_book_list(timeout=timeout)
-        return self.config.get(self.name + '_book_list', [])
+        return self.deseralize_books(self.config.get('book_list', []))
 
+    def seralize_books(self, books):
+        sbooks = []
+        for b in books:
+            data = {}
+            data['author'] = b.author
+            data['title'] = b.title
+            data['detail_item'] = b.detail_item
+            data['formats'] = b.formats
+            sbooks.append(data)
+        return sbooks
 
-class BookRef(SearchResult):
+    def deseralize_books(self, sbooks):
-    
+        books = []
-    def __init__(self):
+        for s in sbooks:
-        SearchResult.__init__(self)
+            b = SearchResult()
-        
+            b.author = s.get('author', '')
-        self.format = ''
+            b.title = s.get('title', '')
+            b.detail_item = s.get('detail_item', '')
+            b.formats = s.get('formats', '')
+            books.append(b)
+        return books
 
 
 class MobeReadStoreDialog(QDialog, Ui_Dialog):
@@ -159,11 +171,11 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog):
             self.plugin.open(self, result.detail_item)
 
     def restore_state(self):
-        geometry = self.plugin.config['store_mobileread_dialog_geometry']
+        geometry = self.plugin.config.get('dialog_geometry', None)
         if geometry:
             self.restoreGeometry(geometry)
 
-        results_cwidth = self.plugin.config['store_mobileread_dialog_results_view_column_width']
+        results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
         if results_cwidth:
             for i, x in enumerate(results_cwidth):
                 if i >= self.results_view.model().columnCount():
@@ -173,16 +185,16 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog):
             for i in xrange(self.results_view.model().columnCount()):
                 self.results_view.resizeColumnToContents(i)
 
-        self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0)
+        self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
-        self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder)
+        self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
         self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
         self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
 
     def save_state(self):
-        self.plugin.config['store_mobileread_dialog_geometry'] = self.saveGeometry()
+        self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
-        self.plugin.config['store_mobileread_dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
+        self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
-        self.plugin.config['store_mobileread_dialog_sort_col'] = self.results_view.model().sort_col
+        self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
-        self.plugin.config['store_mobileread_dialog_sort_order'] = self.results_view.model().sort_order
+        self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
 
     def dialog_closed(self, result):
         self.save_state()
@@ -223,7 +235,7 @@ class BooksModel(QAbstractItemModel):
         self.books = []
         if self.filter:
             for b in self.all_books:
-                test = '%s %s %s' % (b.title, b.author, b.format)
+                test = '%s %s %s' % (b.title, b.author, b.formats)
                 test = test.lower()
                 include = True
                 for item in self.filter.split(' '):
@@ -276,7 +288,7 @@ class BooksModel(QAbstractItemModel):
             elif col == 1:
                 return QVariant(result.author)
             elif col == 2:
-                return QVariant(result.format)
+                return QVariant(result.formats)
         return NONE
 
     def data_as_text(self, result, col):
@@ -286,7 +298,7 @@ class BooksModel(QAbstractItemModel):
         elif col == 1:
             text = result.author
         elif col == 2:
-            text = result.format
+            text = result.formats
         return text
 
     def sort(self, col, order, reset=True):

src/calibre/gui2/store/open_library_plugin.py

@@ -23,10 +23,9 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class OpenLibraryStore(BasicStoreConfig, StorePlugin):
     
     def open(self, parent=None, detail_item=None, external=False):
-        settings = self.get_settings()
         url = 'http://openlibrary.org/'
 
-        if external or settings.get(self.name + '_open_external', False):
+        if external or self.config.get('open_external', False):
             if detail_item:
                 url = url + detail_item
             open_url(QUrl(url_slash_cleaner(url)))
@@ -36,7 +35,7 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin):
                 detail_url = url + detail_item
             d = WebStoreDialog(self.gui, url, parent, detail_url)
             d.setWindowTitle(self.name)
-            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.set_tags(self.config.get('tags', ''))
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
@@ -68,5 +67,15 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price
                 s.detail_item = id.strip()
+                s.drm = SearchResult.DRM_UNKNOWN
                 
                 yield s
+
+    def get_details(self, search_result, timeout):
+        url = 'http://openlibrary.org/'
+        
+        br = browser()
+        with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf:
+            idata = html.fromstring(nf.read())
+            search_result.formats = ', '.join(list(set(idata.xpath('//a[contains(@title, "Download")]/text()'))))
+        return True

src/calibre/gui2/store/search.py

@@ -1,539 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import (unicode_literals, division, absolute_import, print_function)
-
-__license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
-__docformat__ = 'restructuredtext en'
-
-import re
-import time
-import traceback
-from contextlib import closing
-from random import shuffle
-from threading import Thread
-from Queue import Queue
-
-from PyQt4.Qt import (Qt, QAbstractItemModel, QDialog, QTimer, QVariant,
-    QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout)
-
-from calibre import browser
-from calibre.gui2 import NONE
-from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.gui2.store.search_ui import Ui_Dialog
-from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
-    REGEXP_MATCH
-from calibre.utils.config import DynamicConfig
-from calibre.utils.icu import sort_key
-from calibre.utils.magick.draw import thumbnail
-from calibre.utils.search_query_parser import SearchQueryParser
-
-HANG_TIME = 75000 # milliseconds seconds
-TIMEOUT = 75 # seconds
-SEARCH_THREAD_TOTAL = 4
-COVER_DOWNLOAD_THREAD_TOTAL = 2
-
-class SearchDialog(QDialog, Ui_Dialog):
-
-    def __init__(self, istores, *args):
-        QDialog.__init__(self, *args)
-        self.setupUi(self)
-
-        self.config = DynamicConfig('store_search')
-
-        # We keep a cache of store plugins and reference them by name.
-        self.store_plugins = istores
-        self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL)
-        # Check for results and hung threads.
-        self.checker = QTimer()
-        self.hang_check = 0
-
-        self.model = Matches()
-        self.results_view.setModel(self.model)
-
-        # Add check boxes for each store so the user
-        # can disable searching specific stores on a
-        # per search basis.
-        stores_group_layout = QVBoxLayout()
-        self.stores_group.setLayout(stores_group_layout)
-        for x in self.store_plugins:
-            cbox = QCheckBox(x)
-            cbox.setChecked(True)
-            stores_group_layout.addWidget(cbox)
-            setattr(self, 'store_check_' + x, cbox)
-        stores_group_layout.addStretch()
-
-        # Create and add the progress indicator
-        self.pi = ProgressIndicator(self, 24)
-        self.bottom_layout.insertWidget(0, self.pi)
-
-        self.search.clicked.connect(self.do_search)
-        self.checker.timeout.connect(self.get_results)
-        self.results_view.activated.connect(self.open_store)
-        self.select_all_stores.clicked.connect(self.stores_select_all)
-        self.select_invert_stores.clicked.connect(self.stores_select_invert)
-        self.select_none_stores.clicked.connect(self.stores_select_none)
-        self.finished.connect(self.dialog_closed)
-
-        self.restore_state()
-
-    def resize_columns(self):
-        total = 600
-        # Cover
-        self.results_view.setColumnWidth(0, 85)
-        total = total - 85
-        # Title
-        self.results_view.setColumnWidth(1,int(total*.35))
-        # Author
-        self.results_view.setColumnWidth(2,int(total*.35))
-        # Price
-        self.results_view.setColumnWidth(3, int(total*.10))
-        # Store
-        self.results_view.setColumnWidth(4, int(total*.20))
-
-    def do_search(self, checked=False):
-        # Stop all running threads.
-        self.checker.stop()
-        self.search_pool.abort()
-        # Clear the visible results.
-        self.results_view.model().clear_results()
-
-        # Don't start a search if there is nothing to search for.
-        query = unicode(self.search_edit.text())
-        if not query.strip():
-            return
-
-        # Plugins are in alphebetic order. Randomize the
-        # order of plugin names. This way plugins closer
-        # to a don't have an unfair advantage over
-        # plugins further from a.
-        store_names = self.store_plugins.keys()
-        if not store_names:
-            return
-        shuffle(store_names)
-        # Add plugins that the user has checked to the search pool's work queue.
-        for n in store_names:
-            if getattr(self, 'store_check_' + n).isChecked():
-                self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT)
-        if self.search_pool.has_tasks():
-            self.hang_check = 0
-            self.checker.start(100)
-            self.search_pool.start_threads()
-            self.pi.startAnimation()
-
-    def save_state(self):
-        self.config['store_search_geometry'] = self.saveGeometry()
-        self.config['store_search_store_splitter_state'] = self.store_splitter.saveState()
-        self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
-
-        store_check = {}
-        for n in self.store_plugins:
-            store_check[n] = getattr(self, 'store_check_' + n).isChecked()
-        self.config['store_search_store_checked'] = store_check
-
-    def restore_state(self):
-        geometry = self.config['store_search_geometry']
-        if geometry:
-            self.restoreGeometry(geometry)
-
-        splitter_state = self.config['store_search_store_splitter_state']
-        if splitter_state:
-            self.store_splitter.restoreState(splitter_state)
-
-        results_cwidth = self.config['store_search_results_view_column_width']
-        if results_cwidth:
-            for i, x in enumerate(results_cwidth):
-                if i >= self.model.columnCount():
-                    break
-                self.results_view.setColumnWidth(i, x)
-        else:
-            self.resize_columns()
-
-        store_check = self.config['store_search_store_checked']
-        if store_check:
-            for n in store_check:
-                if hasattr(self, 'store_check_' + n):
-                    getattr(self, 'store_check_' + n).setChecked(store_check[n])
-
-    def get_results(self):
-        # We only want the search plugins to run
-        # a maximum set amount of time before giving up.
-        self.hang_check += 1
-        if self.hang_check >= HANG_TIME:
-            self.search_pool.abort()
-            self.checker.stop()
-            self.pi.stopAnimation()
-        else:
-            # Stop the checker if not threads are running.
-            if not self.search_pool.threads_running() and not self.search_pool.has_tasks():
-                self.checker.stop()
-                self.pi.stopAnimation()
-
-        while self.search_pool.has_results():
-            res = self.search_pool.get_result()
-            if res:
-                self.results_view.model().add_result(res)
-
-    def open_store(self, index):
-        result = self.results_view.model().get_result(index)
-        self.store_plugins[result.store_name].open(self, result.detail_item)
-
-    def get_store_checks(self):
-        '''
-        Returns a list of QCheckBox's for each store.
-        '''
-        checks = []
-        for x in self.store_plugins:
-            check = getattr(self, 'store_check_' + x, None)
-            if check:
-                checks.append(check)
-        return checks
-
-    def stores_select_all(self):
-        for check in self.get_store_checks():
-            check.setChecked(True)
-
-    def stores_select_invert(self):
-        for check in self.get_store_checks():
-            check.setChecked(not check.isChecked())
-
-    def stores_select_none(self):
-        for check in self.get_store_checks():
-            check.setChecked(False)
-
-    def dialog_closed(self, result):
-        self.model.closing()
-        self.search_pool.abort()
-        self.save_state()
-
-
-class GenericDownloadThreadPool(object):
-    '''
-    add_task must be implemented in a subclass.
-    '''
-
-    def __init__(self, thread_type, thread_count):
-        self.thread_type = thread_type
-        self.thread_count = thread_count
-
-        self.tasks = Queue()
-        self.results = Queue()
-        self.threads = []
-
-    def add_task(self):
-        raise NotImplementedError()
-
-    def start_threads(self):
-        for i in range(self.thread_count):
-            t = self.thread_type(self.tasks, self.results)
-            self.threads.append(t)
-            t.start()
-
-    def abort(self):
-        self.tasks = Queue()
-        self.results = Queue()
-        for t in self.threads:
-            t.abort()
-        self.threads = []
-
-    def has_tasks(self):
-        return not self.tasks.empty()
-
-    def get_result(self):
-        return self.results.get()
-
-    def get_result_no_wait(self):
-        return self.results.get_nowait()
-
-    def result_count(self):
-        return len(self.results)
-
-    def has_results(self):
-        return not self.results.empty()
-
-    def threads_running(self):
-        for t in self.threads:
-            if t.is_alive():
-                return True
-        return False
-
-
-class SearchThreadPool(GenericDownloadThreadPool):
-    '''
-    Threads will run until there is no work or
-    abort is called. Create and start new threads
-    using start_threads(). Reset by calling abort().
-
-    Example:
-    sp = SearchThreadPool(SearchThread, 3)
-    add tasks using add_task(...)
-    sp.start_threads()
-    all threads have finished.
-    sp.abort()
-    add tasks using add_task(...)
-    sp.start_threads()
-    '''
-
-    def add_task(self, query, store_name, store_plugin, timeout):
-        self.tasks.put((query, store_name, store_plugin, timeout))
-
-
-class SearchThread(Thread):
-
-    def __init__(self, tasks, results):
-        Thread.__init__(self)
-        self.daemon = True
-        self.tasks = tasks
-        self.results = results
-        self._run = True
-
-    def abort(self):
-        self._run = False
-
-    def run(self):
-        while self._run and not self.tasks.empty():
-            try:
-                query, store_name, store_plugin, timeout = self.tasks.get()
-                squery = query
-                for loc in SearchFilter.USABLE_LOCATIONS:
-                    squery = re.sub(r'%s:"?(?P<a>[^\s"]+)"?' % loc, '\g<a>', squery)
-                for res in store_plugin.search(squery, timeout=timeout):
-                    if not self._run:
-                        return
-                    res.store_name = store_name
-                    if SearchFilter(res).parse(query):
-                        self.results.put(res)
-                self.tasks.task_done()
-            except:
-                pass
-
-
-class CoverThreadPool(GenericDownloadThreadPool):
-    '''
-    Once started all threads run until abort is called.
-    '''
-
-    def add_task(self, search_result, update_callback, timeout=5):
-        self.tasks.put((search_result, update_callback, timeout))
-
-
-class CoverThread(Thread):
-
-    def __init__(self, tasks, results):
-        Thread.__init__(self)
-        self.daemon = True
-        self.tasks = tasks
-        self.results = results
-        self._run = True
-
-        self.br = browser()
-
-    def abort(self):
-        self._run = False
-
-    def run(self):
-        while self._run:
-            try:
-                time.sleep(.1)
-                while not self.tasks.empty():
-                    if not self._run:
-                        break
-                    result, callback, timeout = self.tasks.get()
-                    if result and result.cover_url:
-                        with closing(self.br.open(result.cover_url, timeout=timeout)) as f:
-                            result.cover_data = f.read()
-                        result.cover_data = thumbnail(result.cover_data, 64, 64)[2]
-                        callback()
-                    self.tasks.task_done()
-            except:
-                continue
-
-
-class Matches(QAbstractItemModel):
-
-    HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')]
-
-    def __init__(self):
-        QAbstractItemModel.__init__(self)
-        self.matches = []
-        self.cover_pool = CoverThreadPool(CoverThread, 2)
-        self.cover_pool.start_threads()
-
-    def closing(self):
-        self.cover_pool.abort()
-
-    def clear_results(self):
-        self.matches = []
-        self.cover_pool.abort()
-        self.cover_pool.start_threads()
-        self.reset()
-
-    def add_result(self, result):
-        self.layoutAboutToBeChanged.emit()
-        self.matches.append(result)
-        self.cover_pool.add_task(result, self.update_result)
-        self.layoutChanged.emit()
-
-    def get_result(self, index):
-        row = index.row()
-        if row < len(self.matches):
-            return self.matches[row]
-        else:
-            return None
-
-    def update_result(self):
-        self.layoutAboutToBeChanged.emit()
-        self.layoutChanged.emit()
-
-    def index(self, row, column, parent=QModelIndex()):
-        return self.createIndex(row, column)
-
-    def parent(self, index):
-        if not index.isValid() or index.internalId() == 0:
-            return QModelIndex()
-        return self.createIndex(0, 0)
-
-    def rowCount(self, *args):
-        return len(self.matches)
-
-    def columnCount(self, *args):
-        return len(self.HEADERS)
-
-    def headerData(self, section, orientation, role):
-        if role != Qt.DisplayRole:
-            return NONE
-        text = ''
-        if orientation == Qt.Horizontal:
-            if section < len(self.HEADERS):
-                text = self.HEADERS[section]
-            return QVariant(text)
-        else:
-            return QVariant(section+1)
-
-    def data(self, index, role):
-        row, col = index.row(), index.column()
-        result = self.matches[row]
-        if role == Qt.DisplayRole:
-            if col == 1:
-                return QVariant(result.title)
-            elif col == 2:
-                return QVariant(result.author)
-            elif col == 3:
-                return QVariant(result.price)
-            elif col == 4:
-                return QVariant(result.store_name)
-            return NONE
-        elif role == Qt.DecorationRole:
-            if col == 0 and result.cover_data:
-                p = QPixmap()
-                p.loadFromData(result.cover_data)
-                return QVariant(p)
-        elif role == Qt.SizeHintRole:
-            return QSize(64, 64)
-        return NONE
-
-    def data_as_text(self, result, col):
-        text = ''
-        if col == 1:
-            text = result.title
-        elif col == 2:
-            text = result.author
-        elif col == 3:
-            text = result.price
-            if len(text) < 3 or text[-3] not in ('.', ','):
-                text += '00'
-            text = re.sub(r'\D', '', text)
-            text = text.rjust(6, '0')
-        elif col == 4:
-            text = result.store_name
-        return text
-
-    def sort(self, col, order, reset=True):
-        if not self.matches:
-            return
-        descending = order == Qt.DescendingOrder
-        self.matches.sort(None,
-            lambda x: sort_key(unicode(self.data_as_text(x, col))),
-            descending)
-        if reset:
-            self.reset()
-
-
-class SearchFilter(SearchQueryParser):
-    
-    USABLE_LOCATIONS = [
-        'all',
-        'author',
-        'authors',
-        'cover',
-        'price',
-        'title',
-        'store',
-    ]
-
-    def __init__(self, search_result):
-        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
-        self.search_result = search_result
-
-    def universal_set(self):
-        return set([self.search_result])
-
-    def get_matches(self, location, query):
-        location = location.lower().strip()
-        if location == 'authors':
-            location = 'author'
-
-        matchkind = CONTAINS_MATCH
-        if len(query) > 1:
-            if query.startswith('\\'):
-                query = query[1:]
-            elif query.startswith('='):
-                matchkind = EQUALS_MATCH
-                query = query[1:]
-            elif query.startswith('~'):
-                matchkind = REGEXP_MATCH
-                query = query[1:]
-        if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
-            query = query.lower()
-
-        if location not in self.USABLE_LOCATIONS:
-            return set([])
-        matches = set([])
-        all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
-        locations = all_locs if location == 'all' else [location]
-        q = {
-             'author': self.search_result.author.lower(),
-             'cover': self.search_result.cover_url,
-             'format': '',
-             'price': self.search_result.price,
-             'store': self.search_result.store_name.lower(),
-             'title': self.search_result.title.lower(),
-        }
-        for x in ('author', 'format'):
-            q[x+'s'] = q[x]
-        for locvalue in locations:
-            ac_val = q[locvalue]
-            if query == 'true':
-                if ac_val is not None:
-                    matches.add(self.search_result)
-                continue
-            if query == 'false':
-                if ac_val is None:
-                    matches.add(self.search_result)
-                continue
-            try:
-                ### Can't separate authors because comma is used for name sep and author sep
-                ### Exact match might not get what you want. For that reason, turn author
-                ### exactmatch searches into contains searches.
-                if locvalue == 'author' and matchkind == EQUALS_MATCH:
-                    m = CONTAINS_MATCH
-                else:
-                    m = matchkind
-
-                vals = [ac_val]
-                if _match(query, vals, m):
-                    matches.add(self.search_result)
-                    break
-            except ValueError: # Unicode errors
-                traceback.print_exc()
-        return matches