Merge from trunk

Changelog.yaml

@@ -19,6 +19,99 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.7.59
+  date: 2011-04-30
+
+  bug fixes:
+    - title: "Fixes a bug in 0.7.58 that caused too small fonts when converting to MOBI for the Kindle. Apologies."
+
+    - title: "Apple driver: Handle invalid EPUBs that do not contain an OPF file"
+
+  new recipes:
+    - title: The Big Picture and Auto industry news
+      author: welovelucy
+
+    - title: Gazeta Prawna 
+      author: Vroo
+
+    - title: Various Czech news sources
+      author: Tomas Latal
+
+    - title: Diario de Ibiza
+      author: Joan Tur
+
+- version: 0.7.58
+  date: 2011-04-29
+
+  new features:
+    - title: "Support for converting and reading metadata from Plucker format PDB files"
+      type: major
+
+    - title: "The metadata that is displayed in the book details panel on the right is now completely configurable via Preferences->Look & Feel"
+
+    - title: "Add a column that shows the date when the metadata of a book record was last modified in calibre. To see the column, right click on the column headers in calibre and select Show column->Modified. Note that the dates may be incorrect for books added with older versions of calibre."
+
+    - title: "Add command line option to shutdown running calibre"
+
+    - title: "CHM Input: Store extracted files in the input/ sub dir for easy debugging when --debug-pipeline is specified"
+
+    - title: "Add a popup menu to the 'Create saved search button' to allow easy deleting of saved searches"
+  
+  bug fixes:
+    - title: "Fix regression that broke converting to LIT in 0.7.57"
+      tickets: [769334]
+
+    - title: "Conversion pipeline: Remove encoding declarations from input HTML documents to guarantee that there is only a single encoding declaration in the output HTML."
+      tickets: [773337]
+
+    - title: "Correctly parenthesize searches that are used to make search restrictions"
+
+    - title: "Fix ratings in save to disk templates not being divided by 2"
+
+    - title: "TXT to EPUB: Underlined words (following quotes?) fail to become italics"
+      tickets: [772267]
+
+    - title: "Fix template function source code unavailable when not running calibre from source"
+
+    - title: "Fix adding html books from the top of a deep folder hierarchy very slow"
+
+    - title: "Only set language in MOBI metadata if it is not null"
+
+    - title: "Fix 'count-of' searches (e.g., tags:#>3)."
+      tickets: [771175]
+
+    - title: "Fix regression that broke connection to iTunes in some cases"
+      tickets: [771164]
+
+    - title: "Fix buggy regex that made converting PDFs with the string ****************** very slow"
+      tickets: [770534]
+
+    - title: "Fix Ctrl+L shortcut to lookup word not working in ebook viewer"
+      tickets: [769492]
+
+    - title: "Fix regression that broke searching on boolean columns"
+
+  improved recipes:
+    - HBR Blogs
+    - The Marker
+    - Financial Times
+    - Clarin
+    - Honolulu Star Advertiser
+
+  new recipes:
+    - title: Novi Standard
+      author: Darko Miletic
+
+    - title: Autobild.ro and Social Diva
+      author: Silviu Cotoara
+
+    - title: Novinky
+      author: Tomas Latal
+
+    - title: "De Volksrant (subscriber version)"
+      author: Selcal
+
+
 - version: 0.7.57
   date: 2011-04-22
 

recipes/auto_blog.recipe

@@ -0,0 +1,16 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AutoBlog(BasicNewsRecipe):
+    title          = u'Auto Blog'
+    __author__     = 'Welovelucy'
+    language = 'en'
+    description = 'Auto industry news'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds          = [(u'AutoBlog', u'http://www.autoblog.com/rss.xml')]
+
+    def print_version(self, url):
+        return url + 'print/'
+
+

recipes/autobild.recipe

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = u'2011, Silviu Cotoar\u0103'
+'''
+auto-bild.ro
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AutoBild(BasicNewsRecipe):
+    title                       = u'Auto Bild'
+    __author__                   = u'Silviu Cotoar\u0103'
+    description                   = 'Auto'
+    publisher                   = 'Auto Bild'
+    oldest_article              = 50
+    language                    = 'ro'
+    max_articles_per_feed = 100
+    no_stylesheets          = True
+    use_embedded_content  = False
+    category                  = 'Ziare,Reviste,Auto'
+    encoding                = 'utf-8'
+    cover_url     = 'http://www.auto-bild.ro/images/autobild.gif'
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'           : category
+                            ,'language'    : language
+                            ,'publisher'   : publisher
+                         }
+
+
+    keep_only_tags = [
+                dict(name='div', attrs={'class':'box_2 articol clearfix'})
+                 ]
+
+    remove_tags = [
+             dict(name='div', attrs={'class':['detail']})
+           , dict(name='a', attrs={'id':['zoom_link']})
+           , dict(name='div', attrs={'class':['icons clearfix']})
+           , dict(name='div', attrs={'class':['pub_articol clearfix']})
+
+                          ]
+
+    remove_tags_after = [
+              dict(name='div', attrs={'class':['pub_articol clearfix']})
+           ]
+
+    feeds          = [
+        (u'Feeds', u'http://www.auto-bild.ro/rss/toate')
+                     ]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

recipes/big_picture.recipe

@@ -0,0 +1,12 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BigPicture(BasicNewsRecipe):
+    title          = u'The Big Picture'
+    __author__     = 'Welovelucy'
+    description    = ('Macro perspective on capital markets, economy, technology'
+    ' and digital media')
+    language = 'en'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    feeds          = [(u'Big Picture', u'http://feeds.feedburner.com/TheBigPicture')]

recipes/diario_ibiza.recipe

@@ -0,0 +1,55 @@
+__license__   = 'GPL v3'
+__author__    = 'Joan Tur, based on El Pais version by Jordi Balcells & elargentino.com version by Darko Miletic'
+description   = 'Principal periodico de las islas Pitiusas, Ibiza y Formentera (Espanya) - v1.06 (29/04/2011)'
+__docformat__ = 'restructuredtext en'
+
+'''
+diariodeibiza.es
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DiarioDeIbiza(BasicNewsRecipe):
+    __author__        = 'Joan Tur, cullet'
+    description   = 'Principal periodico de las islas Pitiusas, Ibiza y Formentera (Espanya) - v1.06 (29/04/2011)'
+
+    cover_url      = 'http://estaticos01.diariodeibiza.es//elementosWeb/mediaweb/images/logo.jpg'
+    title          = u'Diario de Ibiza digital'
+    publisher      = u'Editorial Prensa Iberica'
+    category       = 'News, politics, culture, economy, general interest'
+    language = 'es'
+
+    encoding              = 'iso-8859-1'
+
+    timefmt        = '[%a, %d %b, %Y]'
+
+    oldest_article = 2
+    max_articles_per_feed = 20
+
+    use_embedded_content  = False
+    recursion             = 5
+
+    remove_javascript = True
+    no_stylesheets = True
+
+    keep_only_tags = [
+                        dict(name='div', attrs={'class':['noticia_titular','epigrafe','subtitulo','actualizada','noticia_fecha','noticia_texto']}),
+                        dict(name='font', attrs={'class':['actualizada']})
+                     ]
+
+    feeds          = [
+                        (u'Portada de Ibiza', u'http://www.diariodeibiza.es/elementosInt/rss/1'),
+                        (u'Pitiuses i Balears', u'http://www.diariodeibiza.es/elementosInt/rss/2'),
+                        (u'Opini\xf3n', u'http://www.diariodeibiza.es/elementosInt/rss/3'),
+                        (u'Nacional', u'http://www.diariodeibiza.es/elementosInt/rss/4'),
+                        (u'Internacional', u'http://www.diariodeibiza.es/elementosInt/rss/5'),
+                        (u'Econom\xeda', u'http://www.diariodeibiza.es/elementosInt/rss/6'),
+                        (u'Deportes', u'http://www.diariodeibiza.es/elementosInt/rss/7'),
+                        (u'Sociedad', u'http://www.diariodeibiza.es/elementosInt/rss/8'),
+                        (u'Ciencia', u'http://www.diariodeibiza.es/elementosInt/rss/11'),
+                        (u'Tecnolog\xeda', u'http://www.diariodeibiza.es/elementosInt/rss/12'),
+                        (u'Gente', u'http://www.diariodeibiza.es/elementosInt/rss/13'),
+                        (u'Sucesos', u'http://www.diariodeibiza.es/elementosInt/rss/15'),
+                        (u'Cultura', u'http://www.diariodeibiza.es/elementosInt/rss/16Piti')
+                    ]
+

recipes/digizone.recipe

@@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Tomas Latal<latal.tomas at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DigiZoneCZ(BasicNewsRecipe):
+    title                   = 'DigiZone'
+    __author__              = 'Tomas Latal'
+    __version__             = '1.0'
+    __date__                = '30 April 2011'
+    description             = u'Aktuality a \u010dl\xe1nky z DigiZone.cz'
+    oldest_article          = 1
+    max_articles_per_feed   = 10
+    encoding                = 'iso-8859-2'
+    publisher               = 'Internet Info s.r.o.'
+    category                = 'digitalni vysilani, televize, CZ'
+    language                = 'cs'
+    publication_type        = 'newsportal'
+    no_stylesheets          = True
+    remove_javascript       = True
+    extra_css               = 'p.perex{font-size: 1.2em; margin: 0 0 10px 0;line-height: 1.4;padding: 0 0 10px 0;font-weight: bold;} \
+                               p.perex img {display:none;} \
+                               .urs p {margin: 0 0 0.8em 0;}'
+
+    feeds          = [
+                        (u'Aktuality', u'http://rss.digizone.cz/aktuality'),
+                        (u'\u010cl\xe1nky', u'http://rss.digizone.cz/clanky')
+                     ]
+
+    remove_tags_before = dict(id=['p-article','p-actuality'])
+
+    remove_tags_after = dict(id=['p-article','p-actuality'])
+
+    remove_tags = [
+                    dict(attrs={'class':['path','mth','lbtr','serial','enquiry','links','dp-n','side','op-ab','op-view','op-sub','op-list',]}),
+                    dict(id=['opinions','discussionList','similarItems','sidebar','footer','opl','promo-box'])
+                  ]

recipes/foxnews.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
 foxnews.com
 '''
@@ -23,6 +23,7 @@ class FoxNews(BasicNewsRecipe):
     extra_css             = """
                                 body{font-family: Arial,sans-serif }
                                 .caption{font-size: x-small}
+                                .author,.dateline{font-size: small}
                             """
 
     conversion_options = {
@@ -34,12 +35,12 @@ class FoxNews(BasicNewsRecipe):
 
     remove_attributes = ['xmlns','lang']
 
-    remove_tags = [
+    remove_tags=[
-                     dict(name=['object','embed','link','script','iframe','meta','base'])
+                  dict(attrs={'class':['user-control','logo','ad-300x250','url-description']})
-                    ,dict(attrs={'class':['user-control','url-description','ad-context']})
+                 ,dict(name=['meta','base','link','iframe','object','embed'])
-                  ]
+                ]
 
-    remove_tags_before=dict(name='h1')
+    keep_only_tags=[dict(attrs={'id':'article-print'})]
     remove_tags_after =dict(attrs={'class':'url-description'})
 
     feeds = [
@@ -55,3 +56,24 @@ class FoxNews(BasicNewsRecipe):
 
     def print_version(self, url):
         return url + 'print'
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+        

recipes/gazeta-prawna-calibre-v1.recipe

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+__license__ = 'GPL v3'
+__copyright__ = u'2011, Vroo <vroobelek@iq.pl>'
+__author__ = u'Vroo'
+'''
+gazetaprawna.pl
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class gazetaprawna(BasicNewsRecipe):
+    version = 1
+    title          = u'Gazeta Prawna'
+    __author__ = u'Vroo'
+    publisher      = u'Infor Biznes'
+    oldest_article = 7
+    max_articles_per_feed = 20
+    no_stylesheets = True
+    remove_javascript = True
+    description = 'Polski dziennik gospodarczy'
+    language = 'pl'
+    encoding = 'utf-8'
+
+    remove_tags_after = [
+               dict(name='div', attrs={'class':['data-art']})
+    ]
+    remove_tags = [
+               dict(name='div', attrs={'class':['dodatki_artykulu','data-art']})
+    ]
+
+    feeds = [
+      (u'Wiadomo\u015bci - najwa\u017cniejsze', u'http://www.gazetaprawna.pl/wiadomosci/najwazniejsze/rss.xml'),
+      (u'Biznes i prawo gospodarcze', u'http://biznes.gazetaprawna.pl/rss.xml'),
+      (u'Prawo i wymiar sprawiedliwo\u015bci', u'http://prawo.gazetaprawna.pl/rss.xml'),
+      (u'Praca i ubezpieczenia', u'http://praca.gazetaprawna.pl/rss.xml'),
+      (u'Podatki i rachunkowo\u015b\u0107', u'http://podatki.gazetaprawna.pl/rss.xml')
+     ]
+
+
+    def print_version(self, url):
+        url = url.replace('wiadomosci/artykuly', 'drukowanie')
+        url = url.replace('artykuly', 'drukowanie')
+        url = url.replace('porady', 'drukowanie')
+        url = url.replace('wywiady', 'drukowanie')
+        url = url.replace('orzeczenia', 'drukowanie')
+        url = url.replace('galeria', 'drukowanie')
+        url = url.replace('komentarze', 'drukowanie')
+        url = url.replace('biznes.gazetaprawna', 'www.gazetaprawna')
+        url = url.replace('podatki.gazetaprawna', 'www.gazetaprawna')
+        url = url.replace('prawo.gazetaprawna', 'www.gazetaprawna')
+        url = url.replace('praca.gazetaprawna', 'www.gazetaprawna')
+        return url

recipes/jezebel.recipe

@@ -16,7 +16,7 @@ class Jezebel(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     encoding              = 'utf-8'
-    use_embedded_content  = False
+    use_embedded_content  = True
     language              = 'en'
     masthead_url          = 'http://cache.gawkerassets.com/assets/jezebel.com/img/logo.png'
     extra_css             = '''
@@ -32,13 +32,12 @@ class Jezebel(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes  = ['width','height']
+    feeds = [(u'Articles', u'http://feeds.gawker.com/jezebel/vip?format=xml')]
-    keep_only_tags     = [dict(attrs={'class':'content permalink'})]
+
-    remove_tags_before = dict(name='h1')
+    remove_tags = [
-    remove_tags        = [dict(attrs={'class':'contactinfo'})]
+            {'class': 'feedflare'},
-    remove_tags_after  = dict(attrs={'class':'contactinfo'})
+    ]
 
-    feeds = [(u'Articles', u'http://feeds.gawker.com/jezebel/full')]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

recipes/kotaku.recipe

@@ -16,7 +16,7 @@ class Kotaku(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     encoding              = 'utf-8'
-    use_embedded_content  = False
+    use_embedded_content  = True
     language              = 'en'
     masthead_url          = 'http://cache.gawkerassets.com/assets/kotaku.com/img/logo.png'
     extra_css             = '''
@@ -31,13 +31,12 @@ class Kotaku(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes  = ['width','height']
+    feeds = [(u'Articles', u'http://feeds.gawker.com/kotaku/vip?format=xml')]
-    keep_only_tags     = [dict(attrs={'class':'content permalink'})]
+
-    remove_tags_before = dict(name='h1')
+    remove_tags = [
-    remove_tags        = [dict(attrs={'class':'contactinfo'})]
+            {'class': 'feedflare'},
-    remove_tags_after  = dict(attrs={'class':'contactinfo'})
+    ]
 
-    feeds = [(u'Articles', u'http://feeds.gawker.com/kotaku/full')]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

recipes/lupa.recipe

@@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Tomas Latal<latal.tomas at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LupaCZ(BasicNewsRecipe):
+    title                   = 'Lupa'
+    __author__              = 'Tomas Latal'
+    __version__             = '1.0'
+    __date__                = '30 April 2011'
+    description             = u'Zpr\xe1vi\u010dky a \u010dl\xe1nky z Lupa.cz'
+    oldest_article          = 2
+    max_articles_per_feed   = 10
+    encoding                = 'utf8'
+    publisher               = 'Internet Info s.r.o.'
+    category                = 'IT,news,CZ'
+    language                = 'cs'
+    publication_type        = 'newsportal'
+    no_stylesheets          = True
+    remove_javascript       = True
+    extra_css               = 'p.perex{font-size: 1.2em;margin: 0 0 10px 0;line-height: 1.4;padding: 0 0 10px 0;font-weight: bold;} \
+                               p.perex img {display:none;} \
+                               .urs p {margin: 0 0 0.8em 0;}'
+
+    feeds          = [
+                        (u'Zpr\xe1vi\u010dky', u'http://rss.lupa.cz/zpravicky'),
+                        (u'\u010cl\xe1nky', u'http://rss.lupa.cz/clanky')
+                    ]
+
+    remove_tags_before = dict(id='main')
+
+    remove_tags_after = [dict(id='main')]
+
+    remove_tags = [
+                    dict(attrs={'class':['author clear','tags-rubrics','box border style1 links clear','enquiry clear','serial','box border style1 TitleList','breadcrumb clear','article-discussion box border style1 monitoringComponentArticle','link-more border prev-next clear']}),
+                    dict(id=['discussionList','similarItems','sidebar','footer','opl','promo-box'])
+                  ]

recipes/mesec.recipe

@@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Tomas Latal<latal.tomas at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class MesecCZ(BasicNewsRecipe):
+    title                   =  u'M\u011b\u0161ec'
+    __author__              = 'Tomas Latal'
+    __version__             = '1.0'
+    __date__                = '30 April 2011'
+    description             = u'Zpr\xe1vi\u010dky a \u010dl\xe1nky z Mesec.cz'
+    oldest_article          = 1
+    max_articles_per_feed   = 10
+    encoding                = 'utf8'
+    publisher               = 'Internet Info s.r.o.'
+    category                = 'finance,CZ'
+    language                = 'cs'
+    publication_type        = 'newsportal'
+    no_stylesheets          = True
+    remove_javascript       = True
+    extra_css               = 'p.perex{font-size: 1.2em;margin: 0 0 10px 0;line-height: 1.4;padding: 0 0 10px 0;font-weight: bold;} \
+                               p.perex img {display:none;} \
+                               .urs p {margin: 0 0 0.8em 0;}'
+
+    feeds          = [
+                        (u'Aktuality', u'http://www.mesec.cz/rss/aktuality/'),
+                        (u'\u010cl\xe1nky', u'http://www.mesec.cz/rss/clanky/')
+                    ]
+
+    remove_tags_before = dict(id='main')
+
+    remove_tags_after = [dict(id='main')]
+
+    remove_tags = [
+                    dict(attrs={'class':['author clear','tags-rubrics','box border style1 links clear','enquiry clear','serial','box border style1 TitleList','breadcrumb clear','article-discussion box border style1 monitoringComponentArticle','link-more border prev-next clear']}),
+                    dict(id=['discussionList','similarItems','sidebar','footer','opl','promo-box'])
+                  ]

recipes/novinky.recipe

@@ -6,8 +6,8 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class NovinkyCZ(BasicNewsRecipe):
     title                   = 'Novinky'
     __author__              = 'Tomas Latal'
-    __version__             = '1.0'
+    __version__             = '1.1'
-    __date__                = '24 April 2011'
+    __date__                = '30 April 2011'
     description             = 'News from server Novinky.cz'
     oldest_article          = 1
     max_articles_per_feed   = 10
@@ -18,6 +18,7 @@ class NovinkyCZ(BasicNewsRecipe):
     publication_type        = 'newsportal'
     no_stylesheets          = True
     remove_javascript       = True
+    cover_url               = 'http://img193.imageshack.us/img193/3039/novinkycover.jpg'
     extra_css               = 'p.acmDescription{font-style:italic;} p.acmAuthor{font-size:0.8em; color:#707070}'
 
     feeds          = [

recipes/novistandard.recipe

@@ -0,0 +1,100 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.standard.rs
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class NoviStandard(BasicNewsRecipe):
+    title                  = 'Novi Standard'
+    __author__             = 'Darko Miletic'
+    description            = 'NoviStandard - energija je neunistiva!'
+    publisher              = 'Novi Standard'
+    category               = 'news, politics, Serbia'
+    no_stylesheets         = True
+    delay                  = 1
+    oldest_article         = 15
+    encoding               = 'utf-8'
+    publication_type       = 'magazine'
+    needs_subscription     = 'optional'
+    remove_empty_feeds     = True
+    INDEX                  = 'http://www.standard.rs/'
+    use_embedded_content   = False
+    language               = 'sr'
+    publication_type       = 'magazine'
+    masthead_url           = 'http://www.standard.rs/templates/ja_opal/images/red/logo.png'
+    extra_css              = """
+                                 @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)}
+                                 @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)}
+                                 body{font-family: Arial,"Segoe UI","Trebuchet MS",Helvetica,sans1,sans-serif}
+                                 .dropcap{font-family: Georgia,Times,serif1,serif; display:inline}
+                                 .dropcap:first-letter{display: inline; font-size: xx-large; font-weight: bold}
+                                 .contentheading{color: gray; font-size: x-large}
+                                 .article-meta, .createdby{color: red}
+                                 img{margin-top:0.5em; margin-bottom: 0.7em; display: block}
+                             """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open(self.INDEX)
+        if self.username is not None and self.password is not None:
+            br.select_form(name='login')
+            br['username'] = self.username
+            br['passwd'  ] = self.password
+            br.submit()
+        return br
+
+    keep_only_tags    =[dict(attrs={'class':['contentheading','article-meta','article-content']})]
+    remove_tags_after =dict(attrs={'class':'extravote-container'})
+    remove_tags       = [
+                           dict(name=['object','link','iframe','meta','base'])
+                          ,dict(attrs={'class':'extravote-container'})
+                        ]
+    remove_attributes =['border','background','height','width','align','valign','lang']
+    feeds             = [
+                            (u'Naslovna', u'http://www.standard.rs/index.php?format=feed&type=rss')
+                           ,(u'Politika', u'http://www.standard.rs/vesti/36-politika.html?format=feed&type=rss')
+                           ,(u'Cvijanovic preporucuje', u'http://www.standard.rs/-cvijanovi-vam-preporuuje.html?format=feed&type=rss')
+                           ,(u'Kolumne', u'http://www.standard.rs/vesti/49-kolumne.html?format=feed&type=rss')
+                           ,(u'Kultura', u'http://www.standard.rs/vesti/40-kultura.html?format=feed&type=rss')
+                           ,(u'Lifestyle', u'http://www.standard.rs/vesti/39-lifestyle.html?format=feed&type=rss')
+                           ,(u'Svet', u'http://www.standard.rs/vesti/41-svet.html?format=feed&type=rss')
+                           ,(u'Ekonomija', u'http://www.standard.rs/vesti/37-ekonomija.html?format=feed&type=rss')
+                           ,(u'Sport', u'http://www.standard.rs/vesti/38-sport.html?format=feed&type=rss')
+                        ]
+
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('div'):
+            if len(item.contents) == 0:
+               item.extract()
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/podnikatel.recipe

@@ -0,0 +1,37 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Tomas Latal<latal.tomas at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class PodnikatelCZ(BasicNewsRecipe):
+    title                   = 'Podnikatel'
+    __author__              = 'Tomas Latal'
+    __version__             = '1.0'
+    __date__                = '30 April 2011'
+    description             = u'Aktuality a \u010dl\xe1nky z Podnikatel.cz'
+    oldest_article          = 1
+    max_articles_per_feed   = 10
+    encoding                = 'utf8'
+    publisher               = 'Internet Info s.r.o.'
+    category                = 'podnikani, bussiness, CZ'
+    language                = 'cs'
+    publication_type        = 'newsportal'
+    no_stylesheets          = True
+    remove_javascript       = True
+    extra_css               = 'p.perex{font-size: 1.2em; margin: 0 0 10px 0;line-height: 1.4;padding: 0 0 10px 0;font-weight: bold;} \
+                               p.perex img {display:none;} \
+                               .urs p {margin: 0 0 0.8em 0;}'
+
+    feeds          = [
+                        (u'Aktuality', u'http://rss.podnikatel.cz/aktuality'),
+                        (u'\u010cl\xe1nky', u'http://rss.podnikatel.cz/clanky')
+                    ]
+
+    remove_tags_before = dict(id='art-content')
+
+    remove_tags_after = [dict(id='art-content')]
+
+    remove_tags = [
+                    dict(attrs={'class':['socialshare','box-blue','author clear','labels-terms','box diskuze','ad','page-nav right','infobox','box zpravy','s-clanky']}),
+                    dict(id=['path','article-tools','discussionList','similarItems','promo-box'])
+                  ]

recipes/vitalia.recipe

@@ -0,0 +1,39 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Tomas Latal<latal.tomas at gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class VitaliaCZ(BasicNewsRecipe):
+    title                   = 'Vitalia'
+    __author__              = 'Tomas Latal'
+    __version__             = '1.0'
+    __date__                = '30 April 2011'
+    description             = u'Aktuality a \u010dl\xe1nky z Vitalia.cz'
+    oldest_article          = 1
+    max_articles_per_feed   = 10
+    encoding                = 'utf8'
+    publisher               = 'Internet Info s.r.o.'
+    category                = 'zdravi, vztahy, wellness, CZ'
+    language                = 'cs'
+    publication_type        = 'newsportal'
+    no_stylesheets          = True
+    remove_javascript       = True
+    extra_css               = 'p.perex{font-size: 1.2em; margin: 0 0 10px 0; line-height: 1.4; padding: 0 0 10px 0; font-weight: bold;} \
+                               p.perex img {display:none;} \
+                               span.author {font-size:0.8em; font-style:italic} \
+                               .urs div.rs-tip-major {padding:0.5em; background: #e0e0e0 none repeat scroll 0 0;border: 1px solid #909090;} \
+                               .urs p {margin: 0 0 0.8em 0;}'
+
+    feeds          = [
+                        (u'Aktuality', 'http://www.vitalia.cz/rss/aktuality/'),
+                        (u'\u010cl\xe1nky', u'http://www.vitalia.cz/rss/clanky/'),
+                    ]
+
+    remove_tags_before = dict(id='main')
+
+    remove_tags_after = [dict(id='main')]
+
+    remove_tags = [
+                    dict(attrs={'class':['author clear','tags-rubrics','box border style1 links clear','enquiry clear','serial','box border style1 TitleList','breadcrumb clear','article-discussion box border style1 monitoringComponentArticle','link-more border prev-next clear']}),
+                    dict(id=['discussionList','similarItems','sidebar','footer','opl','promo-box'])
+                  ]

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 7, 57)
+numeric_version = (0, 7, 59)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \
 from calibre.constants import numeric_version
 from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
-from calibre.utils.config import test_eight_code
 
 # To archive plugins {{{
 class HTML2ZIP(FileTypePlugin):
@@ -621,28 +620,16 @@ from calibre.ebooks.epub.fix.epubcheck import Epubcheck
 plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
         Epubcheck, ]
 
-if test_eight_code:
 # New metadata download plugins {{{
-    from calibre.ebooks.metadata.sources.google import GoogleBooks
+from calibre.ebooks.metadata.sources.google import GoogleBooks
-    from calibre.ebooks.metadata.sources.amazon import Amazon
+from calibre.ebooks.metadata.sources.amazon import Amazon
-    from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
+from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
-    from calibre.ebooks.metadata.sources.isbndb import ISBNDB
+from calibre.ebooks.metadata.sources.isbndb import ISBNDB
-    from calibre.ebooks.metadata.sources.overdrive import OverDrive
+from calibre.ebooks.metadata.sources.overdrive import OverDrive
 
-    plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive]
+plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive]
 
 # }}}
-else:
-    from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
-        KentDistrictLibrary
-    from calibre.ebooks.metadata.douban import DoubanBooks
-    from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers
-    from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
-            AmazonCovers, DoubanCovers
-
-    plugins += [GoogleBooks, ISBNDB, Amazon,
-        OpenLibraryCovers, AmazonCovers, DoubanCovers,
-        NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks]
 
 plugins += [
     ComicInput,
@@ -867,10 +854,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
-        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch]
+        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore]
-
-if test_eight_code:
-    plugins += [ActionStore]
 
 # }}}
 
@@ -1096,10 +1080,8 @@ class Misc(PreferencesPlugin):
 
 plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
         CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
-        Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
+        Email, Server, Plugins, Tweaks, Misc, TemplateFunctions,
-
+        MetadataSources]
-if test_eight_code:
-    plugins.append(MetadataSources)
 
 #}}}
 

src/calibre/customize/ui.py

@@ -15,10 +15,8 @@ from calibre.customize.profiles import InputProfile, OutputProfile
 from calibre.customize.builtins import plugins as builtin_plugins
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.covers import CoverDownload
+from calibre.utils.config import (make_config_dir, Config, ConfigProxy,
-from calibre.ebooks.metadata.fetch import MetadataSource
+                                 plugin_dir, OptionParser)
-from calibre.utils.config import make_config_dir, Config, ConfigProxy, \
-                                 plugin_dir, OptionParser, prefs
 from calibre.ebooks.epub.fix import ePubFixer
 from calibre.ebooks.metadata.sources.base import Source
 
@@ -190,44 +188,6 @@ def output_profiles():
             yield plugin
 # }}}
 
-# Metadata sources {{{
-def metadata_sources(metadata_type='basic', customize=True, isbndb_key=None):
-    for plugin in _initialized_plugins:
-        if isinstance(plugin, MetadataSource) and \
-                plugin.metadata_type == metadata_type:
-            if is_disabled(plugin):
-                continue
-            if customize:
-                customization = config['plugin_customization']
-                plugin.site_customization = customization.get(plugin.name, None)
-            if plugin.name == 'IsbnDB' and isbndb_key is not None:
-                plugin.site_customization = isbndb_key
-            yield plugin
-
-def get_isbndb_key():
-    return config['plugin_customization'].get('IsbnDB', None)
-
-def set_isbndb_key(key):
-    for plugin in _initialized_plugins:
-        if plugin.name == 'IsbnDB':
-            return customize_plugin(plugin, key)
-
-def migrate_isbndb_key():
-    key = prefs['isbndb_com_key']
-    if key:
-        prefs.set('isbndb_com_key', '')
-        set_isbndb_key(key)
-
-def cover_sources():
-    customization = config['plugin_customization']
-    for plugin in _initialized_plugins:
-        if isinstance(plugin, CoverDownload):
-            if not is_disabled(plugin):
-                plugin.site_customization = customization.get(plugin.name, '')
-                yield plugin
-
-# }}}
-
 # Interface Actions # {{{
 
 def interface_actions():

src/calibre/devices/android/driver.py

@@ -109,10 +109,10 @@ class ANDROID(USBMS):
             'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H',
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
-            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE']
+            'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
-            'A70S', 'A101IT', '7', 'INCREDIBLE']
+            'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/apple/driver.py

@@ -163,6 +163,8 @@ class ITUNES(DriverBase):
         settings()
         set_progress_reporter()
         upload_books()
+         _get_fpath()
+          _update_epub_metadata()
         add_books_to_metadata()
         use_plugboard_ext()
         set_plugboard()
@@ -2621,42 +2623,42 @@ class ITUNES(DriverBase):
             # Touch the OPF timestamp
             try:
                 zf_opf = ZipFile(fpath,'r')
+                fnames = zf_opf.namelist()
+                opf = [x for x in fnames if '.opf' in x][0]
             except:
                 raise UserFeedback("'%s' is not a valid EPUB" % metadata.title,
                                    None,
                                    level=UserFeedback.WARN)
-            fnames = zf_opf.namelist()
+
-            opf = [x for x in fnames if '.opf' in x][0]
+            opf_tree = etree.fromstring(zf_opf.read(opf))
-            if opf:
+            md_els = opf_tree.xpath('.//*[local-name()="metadata"]')
-                opf_tree = etree.fromstring(zf_opf.read(opf))
+            if md_els:
-                md_els = opf_tree.xpath('.//*[local-name()="metadata"]')
+                ts = md_els[0].find('.//*[@name="calibre:timestamp"]')
-                if md_els:
+                if ts is not None:
-                    ts = md_els[0].find('.//*[@name="calibre:timestamp"]')
+                    timestamp = ts.get('content')
-                    if ts is not None:
+                    old_ts = parse_date(timestamp)
-                        timestamp = ts.get('content')
+                    metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
-                        old_ts = parse_date(timestamp)
+                                               old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo)
-                        metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
+                    if DEBUG:
-                                                   old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo)
+                        self.log.info("   existing timestamp: %s" % metadata.timestamp)
-                        if DEBUG:
-                            self.log.info("   existing timestamp: %s" % metadata.timestamp)
-                    else:
-                        metadata.timestamp = now()
-                        if DEBUG:
-                            self.log.info("   add timestamp: %s" % metadata.timestamp)
                 else:
                     metadata.timestamp = now()
                     if DEBUG:
-                        self.log.warning("   missing <metadata> block in OPF file")
                         self.log.info("   add timestamp: %s" % metadata.timestamp)
-                # Force the language declaration for iBooks 1.1
+            else:
-                #metadata.language = get_lang().replace('_', '-')
+                metadata.timestamp = now()
-
-                # Updates from metadata plugboard (ignoring publisher)
-                metadata.language = metadata_x.language
-
                 if DEBUG:
-                    if metadata.language != metadata_x.language:
+                    self.log.warning("   missing <metadata> block in OPF file")
-                        self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
+                    self.log.info("   add timestamp: %s" % metadata.timestamp)
+            # Force the language declaration for iBooks 1.1
+            #metadata.language = get_lang().replace('_', '-')
+
+            # Updates from metadata plugboard (ignoring publisher)
+            metadata.language = metadata_x.language
+
+            if DEBUG:
+                if metadata.language != metadata_x.language:
+                    self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
 
             zf_opf.close()
 

src/calibre/devices/misc.py

@@ -187,7 +187,7 @@ class LUMIREAD(USBMS):
             cfilepath = cfilepath.replace(os.sep+'books'+os.sep,
                     os.sep+'covers'+os.sep, 1)
             pdir = os.path.dirname(cfilepath)
-            if not os.exists(pdir):
+            if not os.path.exists(pdir):
                 os.makedirs(pdir)
             with open(cfilepath+'.jpg', 'wb') as f:
                 f.write(metadata.thumbnail[-1])

src/calibre/ebooks/metadata/__init__.py

@@ -274,6 +274,9 @@ def check_isbn(isbn):
     if not isbn:
         return None
     isbn = re.sub(r'[^0-9X]', '', isbn.upper())
+    all_same = re.match(r'(\d)\1{9,12}$', isbn)
+    if all_same is not None:
+        return None
     if len(isbn) == 10:
         return check_isbn10(isbn)
     if len(isbn) == 13:

src/calibre/ebooks/metadata/amazon.py

@@ -1,224 +0,0 @@
-#!/usr/bin/env  python
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
-
-'''
-Fetch metadata using Amazon AWS
-'''
-import sys, re
-from threading import RLock
-
-from lxml import html
-from lxml.html import soupparser
-
-from calibre import browser
-from calibre.ebooks.metadata import check_isbn
-from calibre.ebooks.metadata.book.base import Metadata
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.library.comments import sanitize_comments_html
-
-asin_cache = {}
-cover_url_cache = {}
-cache_lock = RLock()
-
-def find_asin(br, isbn):
-    q = 'http://www.amazon.com/s/?search-alias=aps&field-keywords='+isbn
-    res = br.open_novisit(q)
-    raw = res.read()
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
-            resolve_entities=True)[0]
-    root = html.fromstring(raw)
-    revs = root.xpath('//*[@class="asinReviewsSummary" and @name]')
-    revs = [x.get('name') for x in revs]
-    if revs:
-        return revs[0]
-
-def to_asin(br, isbn):
-    with cache_lock:
-        ans = asin_cache.get(isbn, None)
-    if ans:
-        return ans
-    if ans is False:
-        return None
-    if len(isbn) == 13:
-        try:
-            asin = find_asin(br, isbn)
-        except:
-            import traceback
-            traceback.print_exc()
-            asin = None
-    else:
-        asin = isbn
-    with cache_lock:
-        asin_cache[isbn] = asin if asin else False
-    return asin
-
-
-def get_social_metadata(title, authors, publisher, isbn):
-    mi = Metadata(title, authors)
-    if not isbn:
-        return mi
-    isbn = check_isbn(isbn)
-    if not isbn:
-        return mi
-    br = browser()
-    asin = to_asin(br, isbn)
-    if asin and get_metadata(br, asin, mi):
-        return mi
-    from calibre.ebooks.metadata.xisbn import xisbn
-    for i in xisbn.get_associated_isbns(isbn):
-        asin = to_asin(br, i)
-        if asin and get_metadata(br, asin, mi):
-            return mi
-    return mi
-
-def get_cover_url(isbn, br):
-    isbn = check_isbn(isbn)
-    if not isbn:
-        return None
-    with cache_lock:
-        ans = cover_url_cache.get(isbn, None)
-    if ans:
-        return ans
-    if ans is False:
-        return None
-    asin = to_asin(br, isbn)
-    if asin:
-        ans = _get_cover_url(br, asin)
-        if ans:
-            with cache_lock:
-                cover_url_cache[isbn] = ans
-            return ans
-    from calibre.ebooks.metadata.xisbn import xisbn
-    for i in xisbn.get_associated_isbns(isbn):
-        asin = to_asin(br, i)
-        if asin:
-            ans = _get_cover_url(br, asin)
-            if ans:
-                with cache_lock:
-                    cover_url_cache[isbn] = ans
-                    cover_url_cache[i] = ans
-                return ans
-    with cache_lock:
-        cover_url_cache[isbn] = False
-    return None
-
-def _get_cover_url(br, asin):
-    q = 'http://amzn.com/'+asin
-    try:
-        raw = br.open_novisit(q).read()
-    except Exception as e:
-        if callable(getattr(e, 'getcode', None)) and \
-                e.getcode() == 404:
-            return None
-        raise
-    if '<title>404 - ' in raw:
-        return None
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
-            resolve_entities=True)[0]
-    try:
-        root = soupparser.fromstring(raw)
-    except:
-        return False
-
-    imgs = root.xpath('//img[@id="prodImage" and @src]')
-    if imgs:
-        src = imgs[0].get('src')
-        parts = src.split('/')
-        if len(parts) > 3:
-            bn = parts[-1]
-            sparts = bn.split('_')
-            if len(sparts) > 2:
-                bn = sparts[0] + sparts[-1]
-                return ('/'.join(parts[:-1]))+'/'+bn
-    return None
-
-
-def get_metadata(br, asin, mi):
-    q = 'http://amzn.com/'+asin
-    try:
-        raw = br.open_novisit(q).read()
-    except Exception as e:
-        if callable(getattr(e, 'getcode', None)) and \
-                e.getcode() == 404:
-            return False
-        raise
-    if '<title>404 - ' in raw:
-        return False
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
-            resolve_entities=True)[0]
-    try:
-        root = soupparser.fromstring(raw)
-    except:
-        return False
-    if root.xpath('//*[@id="errorMessage"]'):
-        return False
-
-    ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]')
-    pat = re.compile(r'([0-9.]+) out of (\d+) stars')
-    if ratings:
-        for elem in ratings[0].xpath('descendant::*[@title]'):
-            t = elem.get('title').strip()
-            m = pat.match(t)
-            if m is not None:
-                try:
-                    mi.rating = float(m.group(1))/float(m.group(2)) * 5
-                except:
-                    pass
-
-    desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]')
-    if desc:
-        desc = desc[0]
-        for c in desc.xpath('descendant::*[@class="seeAll" or'
-                ' @class="emptyClear" or @href]'):
-            c.getparent().remove(c)
-        desc = html.tostring(desc, method='html', encoding=unicode).strip()
-        # remove all attributes from tags
-        desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc)
-        # Collapse whitespace
-        #desc = re.sub('\n+', '\n', desc)
-        #desc = re.sub(' +', ' ', desc)
-        # Remove the notice about text referring to out of print editions
-        desc = re.sub(r'(?s)<em>--This text ref.*?</em>', '', desc)
-        # Remove comments
-        desc = re.sub(r'(?s)<!--.*?-->', '', desc)
-        mi.comments = sanitize_comments_html(desc)
-
-    return True
-
-
-def main(args=sys.argv):
-    import tempfile, os
-    tdir = tempfile.gettempdir()
-    br = browser()
-    for title, isbn in [
-            ('The Heroes', '9780316044981'), # Test find_asin
-            ('Learning Python', '8324616489'), # Test xisbn
-            ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting
-            # Random tests
-            ('Star Trek: Destiny: Mere Mortals', '9781416551720'),
-            ('The Great Gatsby', '0743273567'),
-            ]:
-        cpath = os.path.join(tdir, title+'.jpg')
-        curl = get_cover_url(isbn, br)
-        if curl is None:
-            print 'No cover found for', title
-        else:
-            open(cpath, 'wb').write(br.open_novisit(curl).read())
-            print 'Cover for', title, 'saved to', cpath
-
-        #import time
-        #st = time.time()
-        mi = get_social_metadata(title, None, None, isbn)
-        if not mi.comments:
-            print 'Failed to downlaod social metadata for', title
-            return 1
-        #print '\n\n', time.time() - st, '\n\n'
-        print mi
-        print '\n'
-
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/amazonfr.py

@@ -1,516 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2010, sengian <sengian1@gmail.com>'
-
-import sys, textwrap, re, traceback
-from urllib import urlencode
-from math import ceil
-
-from lxml import html
-from lxml.html import soupparser
-
-from calibre.utils.date import parse_date, utcnow, replace_months
-from calibre.utils.cleantext import clean_ascii_chars
-from calibre import browser, preferred_encoding
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.ebooks.metadata import MetaInformation, check_isbn, \
-    authors_to_sort_string
-from calibre.ebooks.metadata.fetch import MetadataSource
-from calibre.utils.config import OptionParser
-from calibre.library.comments import sanitize_comments_html
-
-
-class AmazonFr(MetadataSource):
-
-    name = 'Amazon French'
-    description = _('Downloads metadata from amazon.fr')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    version = (1, 0, 0)
-    has_html_comments = True
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose, lang='fr')
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-class AmazonEs(MetadataSource):
-
-    name = 'Amazon Spanish'
-    description = _('Downloads metadata from amazon.com in spanish')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    version = (1, 0, 0)
-    has_html_comments = True
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose, lang='es')
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-class AmazonEn(MetadataSource):
-
-    name = 'Amazon English'
-    description = _('Downloads metadata from amazon.com in english')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    version = (1, 0, 0)
-    has_html_comments = True
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose, lang='en')
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-class AmazonDe(MetadataSource):
-
-    name = 'Amazon German'
-    description = _('Downloads metadata from amazon.de')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    version = (1, 0, 0)
-    has_html_comments = True
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose, lang='de')
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-class Amazon(MetadataSource):
-
-    name = 'Amazon'
-    description = _('Downloads metadata from amazon.com')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Kovid Goyal & Sengian'
-    version = (1, 1, 0)
-    has_html_comments = True
-
-    def fetch(self):
-        # if not self.site_customization:
-            # return
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose, lang='all')
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    # @property
-    # def string_customization_help(self):
-        # return _('You can select here the language for metadata search with amazon.com')
-
-
-def report(verbose):
-    if verbose:
-        traceback.print_exc()
-
-
-class Query(object):
-
-    BASE_URL_ALL = 'http://www.amazon.com'
-    BASE_URL_FR = 'http://www.amazon.fr'
-    BASE_URL_DE = 'http://www.amazon.de'
-
-    def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None,
-        max_results=20, rlang='all'):
-        assert not(title is None and author is None and publisher is None \
-            and isbn is None and keywords is None)
-        assert (max_results < 21)
-
-        self.max_results = int(max_results)
-        self.renbres = re.compile(u'\s*(\d+)\s*')
-
-        q = {   'search-alias' : 'stripbooks' ,
-                'unfiltered' : '1',
-                'field-keywords' : '',
-                'field-author' : '',
-                'field-title' : '',
-                'field-isbn' : '',
-                'field-publisher' : ''
-                #get to amazon detailed search page to get all options
-                # 'node' : '',
-                # 'field-binding' : '',
-                #before, during, after
-                # 'field-dateop' : '',
-                #month as number
-                # 'field-datemod' : '',
-                # 'field-dateyear' : '',
-                #french only
-                # 'field-collection' : '',
-                #many options available
-            }
-
-        if rlang =='all':
-            q['sort'] = 'relevanceexprank'
-            self.urldata = self.BASE_URL_ALL
-        elif rlang =='es':
-            q['sort'] = 'relevanceexprank'
-            q['field-language'] = 'Spanish'
-            self.urldata = self.BASE_URL_ALL
-        elif rlang =='en':
-            q['sort'] = 'relevanceexprank'
-            q['field-language'] = 'English'
-            self.urldata = self.BASE_URL_ALL
-        elif rlang =='fr':
-            q['sort'] = 'relevancerank'
-            self.urldata = self.BASE_URL_FR
-        elif rlang =='de':
-            q['sort'] = 'relevancerank'
-            self.urldata = self.BASE_URL_DE
-        self.baseurl = self.urldata
-
-        if isbn is not None:
-            q['field-isbn'] = isbn.replace('-', '')
-        else:
-            if title is not None:
-                q['field-title'] = title
-            if author is not None:
-                q['field-author'] = author
-            if publisher is not None:
-                q['field-publisher'] = publisher
-            if keywords is not None:
-                q['field-keywords'] = keywords
-
-        if isinstance(q, unicode):
-            q = q.encode('utf-8')
-        self.urldata += '/gp/search/ref=sr_adv_b/?' + urlencode(q)
-
-    def __call__(self, browser, verbose, timeout = 5.):
-        if verbose:
-            print 'Query:', self.urldata
-
-        try:
-            raw = browser.open_novisit(self.urldata, timeout=timeout).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            raise
-        if '<title>404 - ' in raw:
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-
-        try:
-            feed = soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                return soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                return None, self.urldata
-
-        #nb of page
-        try:
-            nbresults = self.renbres.findall(feed.xpath("//*[@class='resultCount']")[0].text)
-        except:
-            return None, self.urldata
-
-        pages =[feed]
-        if len(nbresults) > 1:
-            nbpagetoquery = int(ceil(float(min(int(nbresults[2]), self.max_results))/ int(nbresults[1])))
-            for i in xrange(2, nbpagetoquery + 1):
-                try:
-                    urldata = self.urldata + '&page=' + str(i)
-                    raw = browser.open_novisit(urldata, timeout=timeout).read()
-                except Exception as e:
-                    continue
-                if '<title>404 - ' in raw:
-                    continue
-                raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                        resolve_entities=True)[0]
-                try:
-                    feed = soupparser.fromstring(raw)
-                except:
-                    try:
-                        #remove ASCII invalid chars
-                        return soupparser.fromstring(clean_ascii_chars(raw))
-                    except:
-                        continue
-                pages.append(feed)
-
-        results = []
-        for x in pages:
-            results.extend([i.getparent().get('href') \
-                for i in x.xpath("//a/span[@class='srTitle']")])
-        return results[:self.max_results], self.baseurl
-
-class ResultList(list):
-
-    def __init__(self, baseurl, lang = 'all'):
-        self.baseurl = baseurl
-        self.lang = lang
-        self.repub = re.compile(u'\((.*)\)')
-        self.rerat = re.compile(u'([0-9.]+)')
-        self.reattr = re.compile(r'<([a-zA-Z0-9]+)\s[^>]+>')
-        self.reoutp = re.compile(r'(?s)<em>--This text ref.*?</em>')
-        self.recom = re.compile(r'(?s)<!--.*?-->')
-        self.republi = re.compile(u'(Editeur|Publisher|Verlag)', re.I)
-        self.reisbn = re.compile(u'(ISBN-10|ISBN-10|ASIN)', re.I)
-        self.relang = re.compile(u'(Language|Langue|Sprache)', re.I)
-        self.reratelt = re.compile(u'(Average\s*Customer\s*Review|Moyenne\s*des\s*commentaires\s*client|Durchschnittliche\s*Kundenbewertung)', re.I)
-        self.reprod = re.compile(u'(Product\s*Details|D.tails\s*sur\s*le\s*produit|Produktinformation)', re.I)
-
-    def strip_tags_etree(self, etreeobj, invalid_tags):
-        for (itag, rmv) in invalid_tags.iteritems():
-            if rmv:
-                for elts in etreeobj.getiterator(itag):
-                    elts.drop_tree()
-            else:
-                for elts in etreeobj.getiterator(itag):
-                    elts.drop_tag()
-
-    def clean_entry(self, entry, invalid_tags = {'script': True},
-                invalid_id = (), invalid_class=()):
-        #invalid_tags: remove tag and keep content if False else remove
-        #remove tags
-        if invalid_tags:
-            self.strip_tags_etree(entry, invalid_tags)
-        #remove id
-        if invalid_id:
-            for eltid in invalid_id:
-                elt = entry.get_element_by_id(eltid)
-                if elt is not None:
-                    elt.drop_tree()
-        #remove class
-        if invalid_class:
-            for eltclass in invalid_class:
-                elts = entry.find_class(eltclass)
-                if elts is not None:
-                    for elt in elts:
-                        elt.drop_tree()
-
-    def get_title(self, entry):
-        title = entry.get_element_by_id('btAsinTitle')
-        if title is not None:
-            title = title.text
-        return unicode(title.replace('\n', '').strip())
-
-    def get_authors(self, entry):
-        author = entry.get_element_by_id('btAsinTitle')
-        while author.getparent().tag != 'div':
-            author = author.getparent()
-        author = author.getparent()
-        authortext = []
-        for x in author.getiterator('a'):
-            authortext.append(unicode(x.text_content().strip()))
-        return authortext
-
-    def get_description(self, entry, verbose):
-        try:
-            description = entry.get_element_by_id("productDescription").find("div[@class='content']")
-            inv_class = ('seeAll', 'emptyClear')
-            inv_tags ={'img': True, 'a': False}
-            self.clean_entry(description, invalid_tags=inv_tags, invalid_class=inv_class)
-            description = html.tostring(description, method='html', encoding=unicode).strip()
-            # remove all attributes from tags
-            description = self.reattr.sub(r'<\1>', description)
-            # Remove the notice about text referring to out of print editions
-            description = self.reoutp.sub('', description)
-            # Remove comments
-            description = self.recom.sub('', description)
-            return unicode(sanitize_comments_html(description))
-        except:
-            report(verbose)
-            return None
-
-    def get_tags(self, entry, browser, verbose):
-        try:
-            tags = entry.get_element_by_id('tagContentHolder')
-            testptag = tags.find_class('see-all')
-            if testptag:
-                for x in testptag:
-                    alink = x.xpath('descendant-or-self::a')
-                    if alink:
-                        if alink[0].get('class') == 'tgJsActive':
-                            continue
-                        link = self.baseurl + alink[0].get('href')
-                        entry = self.get_individual_metadata(browser, link, verbose)
-                        tags = entry.get_element_by_id('tagContentHolder')
-                        break
-            tags = [a.text for a in tags.getiterator('a') if a.get('rel') == 'tag']
-        except:
-            report(verbose)
-            tags = []
-        return tags
-
-    def get_book_info(self, entry, mi, verbose):
-        try:
-            entry = entry.get_element_by_id('SalesRank').getparent()
-        except:
-            try:
-                for z in entry.getiterator('h2'):
-                    if self.reprod.search(z.text_content()):
-                        entry = z.getparent().find("div[@class='content']/ul")
-                        break
-            except:
-                report(verbose)
-                return mi
-        elts = entry.findall('li')
-        #pub & date
-        elt = filter(lambda x: self.republi.search(x.find('b').text), elts)
-        if elt:
-            pub = elt[0].find('b').tail
-            mi.publisher = unicode(self.repub.sub('', pub).strip())
-            d = self.repub.search(pub)
-            if d is not None:
-                d = d.group(1)
-                try:
-                    default = utcnow().replace(day=15)
-                    if self.lang != 'all':
-                        d = replace_months(d, self.lang)
-                    d = parse_date(d, assume_utc=True, default=default)
-                    mi.pubdate = d
-                except:
-                    report(verbose)
-        #ISBN
-        elt = filter(lambda x: self.reisbn.search(x.find('b').text), elts)
-        if elt:
-            isbn = elt[0].find('b').tail.replace('-', '').strip()
-            if check_isbn(isbn):
-                    mi.isbn = unicode(isbn)
-            elif len(elt) > 1:
-                isbn = elt[1].find('b').tail.replace('-', '').strip()
-                if check_isbn(isbn):
-                    mi.isbn = unicode(isbn)
-        #Langue
-        elt = filter(lambda x: self.relang.search(x.find('b').text), elts)
-        if elt:
-            langue = elt[0].find('b').tail.strip()
-            if langue:
-                mi.language = unicode(langue)
-        #ratings
-        elt = filter(lambda x: self.reratelt.search(x.find('b').text), elts)
-        if elt:
-            ratings = elt[0].find_class('swSprite')
-            if ratings:
-                ratings = self.rerat.findall(ratings[0].get('title'))
-                if len(ratings) == 2:
-                    mi.rating = float(ratings[0])/float(ratings[1]) * 5
-        return mi
-
-    def fill_MI(self, entry, title, authors, browser, verbose):
-        mi = MetaInformation(title, authors)
-        mi.author_sort = authors_to_sort_string(authors)
-        mi.comments = self.get_description(entry, verbose)
-        mi = self.get_book_info(entry, mi, verbose)
-        mi.tags = self.get_tags(entry, browser, verbose)
-        return mi
-
-    def get_individual_metadata(self, browser, linkdata, verbose):
-        try:
-            raw = browser.open_novisit(linkdata).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            raise
-        if '<title>404 - ' in raw:
-            report(verbose)
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-        try:
-            return soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                return soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                report(verbose)
-                return
-
-    def populate(self, entries, browser, verbose=False):
-        for x in entries:
-            try:
-                entry = self.get_individual_metadata(browser, x, verbose)
-                # clean results
-                # inv_ids = ('divsinglecolumnminwidth', 'sims.purchase', 'AutoBuyXGetY', 'A9AdsMiddleBoxTop')
-                # inv_class = ('buyingDetailsGrid', 'productImageGrid')
-                # inv_tags ={'script': True, 'style': True, 'form': False}
-                # self.clean_entry(entry, invalid_id=inv_ids)
-                title = self.get_title(entry)
-                authors = self.get_authors(entry)
-            except Exception as e:
-                if verbose:
-                    print 'Failed to get all details for an entry'
-                    print e
-                    print 'URL who failed:', x
-                    report(verbose)
-                continue
-            self.append(self.fill_MI(entry, title, authors, browser, verbose))
-
-
-def search(title=None, author=None, publisher=None, isbn=None,
-           max_results=5, verbose=False, keywords=None, lang='all'):
-    br = browser()
-    entries, baseurl = Query(title=title, author=author, isbn=isbn, publisher=publisher,
-        keywords=keywords, max_results=max_results,rlang=lang)(br, verbose)
-
-    if entries is None or len(entries) == 0:
-        return
-
-    #List of entry
-    ans = ResultList(baseurl, lang)
-    ans.populate(entries, br, verbose)
-    return ans
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(\
-    _('''\
-        %prog [options]
-
-        Fetch book metadata from Amazon. You must specify one of title, author,
-        ISBN, publisher or keywords. Will fetch a maximum of 10 matches,
-        so you should make your query as specific as possible.
-        You can chose the language for metadata retrieval:
-        All & english & french & german & spanish
-    '''
-    )))
-    parser.add_option('-t', '--title', help='Book title')
-    parser.add_option('-a', '--author', help='Book author(s)')
-    parser.add_option('-p', '--publisher', help='Book publisher')
-    parser.add_option('-i', '--isbn', help='Book ISBN')
-    parser.add_option('-k', '--keywords', help='Keywords')
-    parser.add_option('-m', '--max-results', default=10,
-                      help='Maximum number of results to fetch')
-    parser.add_option('-l', '--lang', default='all',
-                      help='Chosen language for metadata search (all, en, fr, es, de)')
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help='Be more verbose about errors')
-    return parser
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    try:
-        results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher,
-            keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results,
-                lang=opts.lang)
-    except AssertionError:
-        report(True)
-        parser.print_help()
-        return 1
-    if results is None or len(results) == 0:
-        print 'No result found for this search!'
-        return 0
-    for result in results:
-        print unicode(result).encode(preferred_encoding, 'replace')
-        print
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/covers.py

@@ -1,317 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
-__license__   = 'GPL v3'
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import traceback, socket, sys
-from functools import partial
-from threading import Thread, Event
-from Queue import Queue, Empty
-from lxml import etree
-
-import mechanize
-
-from calibre.customize import Plugin
-from calibre import browser, prints
-from calibre.constants import preferred_encoding, DEBUG
-
-class CoverDownload(Plugin):
-    '''
-    These plugins are used to download covers for books.
-    '''
-
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Kovid Goyal'
-    type = _('Cover download')
-
-    def has_cover(self, mi, ans, timeout=5.):
-        '''
-        Check if the book described by mi has a cover. Call ans.set() if it
-        does. Do nothing if it doesn't.
-
-        :param mi: MetaInformation object
-        :param timeout: timeout in seconds
-        :param ans: A threading.Event object
-        '''
-        raise NotImplementedError()
-
-    def get_covers(self, mi, result_queue, abort, timeout=5.):
-        '''
-        Download covers for books described by the mi object. Downloaded covers
-        must be put into the result_queue. If more than one cover is available,
-        the plugin should continue downloading them and putting them into
-        result_queue until abort.is_set() returns True.
-
-        :param mi: MetaInformation object
-        :param result_queue: A multithreaded Queue
-        :param abort: A threading.Event object
-        :param timeout: timeout in seconds
-        '''
-        raise NotImplementedError()
-
-    def exception_to_string(self, ex):
-        try:
-            return unicode(ex)
-        except:
-            try:
-                return str(ex).decode(preferred_encoding, 'replace')
-            except:
-                return repr(ex)
-
-    def debug(self, *args, **kwargs):
-        if DEBUG:
-            prints('\t'+self.name+':', *args, **kwargs)
-
-
-
-class HeadRequest(mechanize.Request):
-
-    def get_method(self):
-        return 'HEAD'
-
-class OpenLibraryCovers(CoverDownload): # {{{
-    'Download covers from openlibrary.org'
-
-    # See http://openlibrary.org/dev/docs/api/covers
-
-    OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
-    name = 'openlibrary.org covers'
-    description = _('Download covers from openlibrary.org')
-    author = 'Kovid Goyal'
-
-    def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
-            return False
-        from calibre.ebooks.metadata.library_thing import get_browser
-        br = get_browser()
-        br.set_handle_redirect(False)
-        try:
-            br.open_novisit(HeadRequest(self.OPENLIBRARY%mi.isbn), timeout=timeout)
-            self.debug('cover for', mi.isbn, 'found')
-            ans.set()
-        except Exception as e:
-            if callable(getattr(e, 'getcode', None)) and e.getcode() == 302:
-                self.debug('cover for', mi.isbn, 'found')
-                ans.set()
-            else:
-                self.debug(e)
-
-    def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
-            return
-        from calibre.ebooks.metadata.library_thing import get_browser
-        br = get_browser()
-        try:
-            ans = br.open(self.OPENLIBRARY%mi.isbn, timeout=timeout).read()
-            result_queue.put((True, ans, 'jpg', self.name))
-        except Exception as e:
-            if callable(getattr(e, 'getcode', None)) and e.getcode() == 404:
-                result_queue.put((False, _('ISBN: %s not found')%mi.isbn, '', self.name))
-            else:
-                result_queue.put((False, self.exception_to_string(e),
-                    traceback.format_exc(), self.name))
-
-# }}}
-
-class AmazonCovers(CoverDownload): # {{{
-
-    name = 'amazon.com covers'
-    description = _('Download covers from amazon.com')
-    author = 'Kovid Goyal'
-
-
-    def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
-            return False
-        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = browser()
-        try:
-            get_cover_url(mi.isbn, br)
-            self.debug('cover for', mi.isbn, 'found')
-            ans.set()
-        except Exception as e:
-            self.debug(e)
-
-    def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
-            return
-        from calibre.ebooks.metadata.amazon import get_cover_url
-        br = browser()
-        try:
-            url = get_cover_url(mi.isbn, br)
-            if url is None:
-                raise ValueError('No cover found for ISBN: %s'%mi.isbn)
-            cover_data = br.open_novisit(url).read()
-            result_queue.put((True, cover_data, 'jpg', self.name))
-        except Exception as e:
-            result_queue.put((False, self.exception_to_string(e),
-                traceback.format_exc(), self.name))
-
-# }}}
-
-def check_for_cover(mi, timeout=5.): # {{{
-    from calibre.customize.ui import cover_sources
-    ans = Event()
-    checkers = [partial(p.has_cover, mi, ans, timeout=timeout) for p in
-            cover_sources()]
-    workers = [Thread(target=c) for c in checkers]
-    for w in workers:
-        w.daemon = True
-        w.start()
-    while not ans.is_set():
-        ans.wait(0.1)
-        if sum([int(w.is_alive()) for w in workers]) == 0:
-            break
-    return ans.is_set()
-
-# }}}
-
-def download_covers(mi, result_queue, max_covers=50, timeout=5.): # {{{
-    from calibre.customize.ui import cover_sources
-    abort = Event()
-    temp = Queue()
-    getters = [partial(p.get_covers, mi, temp, abort, timeout=timeout) for p in
-            cover_sources()]
-    workers = [Thread(target=c) for c in getters]
-    for w in workers:
-        w.daemon = True
-        w.start()
-    count = 0
-    while count < max_covers:
-        try:
-            result = temp.get_nowait()
-            if result[0]:
-                count += 1
-            result_queue.put(result)
-        except Empty:
-            pass
-        if sum([int(w.is_alive()) for w in workers]) == 0:
-            break
-
-    abort.set()
-
-    while True:
-        try:
-            result = temp.get_nowait()
-            count += 1
-            result_queue.put(result)
-        except Empty:
-            break
-
-# }}}
-
-class DoubanCovers(CoverDownload): # {{{
-    'Download covers from Douban.com'
-
-    DOUBAN_ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
-    CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
-    name = 'Douban.com covers'
-    description = _('Download covers from Douban.com')
-    author = 'Li Fanxi'
-
-    def get_cover_url(self, isbn, br, timeout=5.):
-        try:
-            url = self.DOUBAN_ISBN_URL + isbn + "?apikey=" + self.CALIBRE_DOUBAN_API_KEY
-            src = br.open(url, timeout=timeout).read()
-        except Exception as err:
-            if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-                err = Exception(_('Douban.com API timed out. Try again later.'))
-            raise err
-        else:
-            feed = etree.fromstring(src)
-            NAMESPACES = {
-              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
-              'atom' : 'http://www.w3.org/2005/Atom',
-              'db': 'http://www.douban.com/xmlns/'
-            }
-            XPath = partial(etree.XPath, namespaces=NAMESPACES)
-            entries = XPath('//atom:entry')(feed)
-            if len(entries) < 1:
-                return None
-            try:
-                cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
-                u = cover_url(entries[0])[0].replace('/spic/', '/lpic/');
-                # If URL contains "book-default", the book doesn't have a cover
-                if u.find('book-default') != -1:
-                    return None
-            except:
-                return None
-            return u
-
-    def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
-            return False
-        br = browser()
-        try:
-            if self.get_cover_url(mi.isbn, br, timeout=timeout) != None:
-                self.debug('cover for', mi.isbn, 'found')
-                ans.set()
-        except Exception as e:
-            self.debug(e)
-
-    def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
-            return
-        br = browser()
-        try:
-            url = self.get_cover_url(mi.isbn, br, timeout=timeout)
-            cover_data = br.open_novisit(url).read()
-            result_queue.put((True, cover_data, 'jpg', self.name))
-        except Exception as e:
-            result_queue.put((False, self.exception_to_string(e),
-                traceback.format_exc(), self.name))
-# }}}
-
-def download_cover(mi, timeout=5.): # {{{
-    results = Queue()
-    download_covers(mi, results, max_covers=1, timeout=timeout)
-    errors, ans = [], None
-    while True:
-        try:
-            x = results.get_nowait()
-            if x[0]:
-                ans = x[1]
-            else:
-                errors.append(x)
-        except Empty:
-            break
-    return ans, errors
-
-# }}}
-
-def test(isbns): # {{{
-    from calibre.ebooks.metadata import MetaInformation
-    mi = MetaInformation('test', ['test'])
-    for isbn in isbns:
-        prints('Testing ISBN:', isbn)
-        mi.isbn = isbn
-        found = check_for_cover(mi)
-        prints('Has cover:', found)
-        ans, errors = download_cover(mi)
-        if ans is not None:
-            prints('Cover downloaded')
-        else:
-            prints('Download failed:')
-            for err in errors:
-                prints('\t', err[-1]+':', err[1])
-        print '\n'
-
-# }}}
-
-if __name__ == '__main__':
-    isbns = sys.argv[1:] + ['9781591025412', '9780307272119']
-    #test(isbns)
-
-    from calibre.ebooks.metadata import MetaInformation
-    oc = OpenLibraryCovers(None)
-    for isbn in isbns:
-        mi = MetaInformation('xx', ['yy'])
-        mi.isbn = isbn
-        rq = Queue()
-        oc.get_covers(mi, rq, Event())
-        result = rq.get_nowait()
-        if not result[0]:
-            print 'Failed for ISBN:', isbn
-            print result

src/calibre/ebooks/metadata/douban.py

@@ -1,263 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>; 2010, Li Fanxi <lifanxi@freemindworld.com>'
-__docformat__ = 'restructuredtext en'
-
-import sys, textwrap
-import traceback
-from urllib import urlencode
-from functools import partial
-from lxml import etree
-
-from calibre import browser, preferred_encoding
-from calibre.ebooks.metadata import MetaInformation
-from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata.fetch import MetadataSource
-from calibre.utils.date import parse_date, utcnow
-
-NAMESPACES = {
-              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
-              'atom' : 'http://www.w3.org/2005/Atom',
-              'db': 'http://www.douban.com/xmlns/'
-            }
-XPath = partial(etree.XPath, namespaces=NAMESPACES)
-total_results  = XPath('//openSearch:totalResults')
-start_index    = XPath('//openSearch:startIndex')
-items_per_page = XPath('//openSearch:itemsPerPage')
-entry          = XPath('//atom:entry')
-entry_id       = XPath('descendant::atom:id')
-title          = XPath('descendant::atom:title')
-description    = XPath('descendant::atom:summary')
-publisher      = XPath("descendant::db:attribute[@name='publisher']")
-isbn           = XPath("descendant::db:attribute[@name='isbn13']")
-date           = XPath("descendant::db:attribute[@name='pubdate']")
-creator        = XPath("descendant::db:attribute[@name='author']")
-tag            = XPath("descendant::db:tag")
-
-CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
-
-class DoubanBooks(MetadataSource):
-
-    name = 'Douban Books'
-    description = _('Downloads metadata from Douban.com')
-    supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on
-    author              = 'Li Fanxi <lifanxi@freemindworld.com>' # The author of this plugin
-    version             = (1, 0, 1)   # The version number of this plugin
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10,
-                                  verbose=self.verbose)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-def report(verbose):
-    if verbose:
-        import traceback
-        traceback.print_exc()
-
-class Query(object):
-
-    SEARCH_URL = 'http://api.douban.com/book/subjects?'
-    ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
-
-    type = "search"
-
-    def __init__(self, title=None, author=None, publisher=None, isbn=None,
-                 max_results=20, start_index=1, api_key=''):
-        assert not(title is None and author is None and publisher is None and \
-                   isbn is None)
-        assert (int(max_results) < 21)
-        q = ''
-        if isbn is not None:
-            q = isbn
-            self.type = 'isbn'
-        else:
-            def build_term(parts):
-                return ' '.join(x for x in parts)
-            if title is not None:
-                q += build_term(title.split())
-            if author is not None:
-                q += (' ' if q else '') + build_term(author.split())
-            if publisher is not None:
-                q += (' ' if q else '') + build_term(publisher.split())
-            self.type = 'search'
-
-        if isinstance(q, unicode):
-            q = q.encode('utf-8')
-
-        if self.type == "isbn":
-            self.url = self.ISBN_URL + q
-            if api_key != '':
-                self.url = self.url + "?apikey=" + api_key
-        else:
-            self.url = self.SEARCH_URL+urlencode({
-                    'q':q,
-                    'max-results':max_results,
-                    'start-index':start_index,
-                    })
-            if api_key != '':
-                self.url = self.url + "&apikey=" + api_key
-
-    def __call__(self, browser, verbose):
-        if verbose:
-            print 'Query:', self.url
-        if self.type == "search":
-            feed = etree.fromstring(browser.open(self.url).read())
-            total = int(total_results(feed)[0].text)
-            start = int(start_index(feed)[0].text)
-            entries = entry(feed)
-            new_start = start + len(entries)
-            if new_start > total:
-                new_start = 0
-            return entries, new_start
-        elif self.type == "isbn":
-            feed = etree.fromstring(browser.open(self.url).read())
-            entries = entry(feed)
-            return entries, 0
-
-class ResultList(list):
-
-    def get_description(self, entry, verbose):
-        try:
-            desc = description(entry)
-            if desc:
-                return 'SUMMARY:\n'+desc[0].text
-        except:
-            report(verbose)
-
-    def get_title(self, entry):
-        candidates = [x.text for x in title(entry)]
-        return ': '.join(candidates)
-
-    def get_authors(self, entry):
-        m = creator(entry)
-        if not m:
-            m = []
-        m = [x.text for x in m]
-        return m
-
-    def get_tags(self, entry, verbose):
-        try:
-            btags = [x.attrib["name"] for x in tag(entry)]
-            tags = []
-            for t in btags:
-                tags.extend([y.strip() for y in t.split('/')])
-            tags = list(sorted(list(set(tags))))
-        except:
-            report(verbose)
-            tags = []
-        return [x.replace(',', ';') for x in tags]
-
-    def get_publisher(self, entry, verbose):
-        try:
-            pub = publisher(entry)[0].text
-        except:
-            pub = None
-        return pub
-
-    def get_isbn(self, entry, verbose):
-        try:
-            isbn13 = isbn(entry)[0].text
-        except Exception:
-            isbn13 = None
-        return isbn13
-
-    def get_date(self, entry, verbose):
-        try:
-            d = date(entry)
-            if d:
-                default = utcnow().replace(day=15)
-                d = parse_date(d[0].text, assume_utc=True, default=default)
-            else:
-                d = None
-        except:
-            report(verbose)
-            d = None
-        return d
-
-    def populate(self, entries, browser, verbose=False, api_key=''):
-        for x in entries:
-            try:
-                id_url = entry_id(x)[0].text
-                title = self.get_title(x)
-            except:
-                report(verbose)
-            mi = MetaInformation(title, self.get_authors(x))
-            try:
-                if api_key != '':
-                    id_url = id_url + "?apikey=" + api_key
-                raw = browser.open(id_url).read()
-                feed = etree.fromstring(raw)
-                x = entry(feed)[0]
-            except Exception as e:
-                if verbose:
-                    print 'Failed to get all details for an entry'
-                    print e
-            mi.comments = self.get_description(x, verbose)
-            mi.tags = self.get_tags(x, verbose)
-            mi.isbn = self.get_isbn(x, verbose)
-            mi.publisher = self.get_publisher(x, verbose)
-            mi.pubdate = self.get_date(x, verbose)
-            self.append(mi)
-
-def search(title=None, author=None, publisher=None, isbn=None,
-           verbose=False, max_results=40, api_key=None):
-    br   = browser()
-    start, entries = 1, []
-
-    if api_key is None:
-        api_key = CALIBRE_DOUBAN_API_KEY
-
-    while start > 0 and len(entries) <= max_results:
-        new, start = Query(title=title, author=author, publisher=publisher,
-                       isbn=isbn, max_results=max_results, start_index=start, api_key=api_key)(br, verbose)
-        if not new:
-            break
-        entries.extend(new)
-
-    entries = entries[:max_results]
-
-    ans = ResultList()
-    ans.populate(entries, br, verbose, api_key)
-    return ans
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(
-        '''\
-        %prog [options]
-
-        Fetch book metadata from Douban. You must specify one of title, author,
-        publisher or ISBN. If you specify ISBN the others are ignored. Will
-        fetch a maximum of 100 matches, so you should make your query as
-        specific as possible.
-        '''
-    ))
-    parser.add_option('-t', '--title', help='Book title')
-    parser.add_option('-a', '--author', help='Book author(s)')
-    parser.add_option('-p', '--publisher', help='Book publisher')
-    parser.add_option('-i', '--isbn', help='Book ISBN')
-    parser.add_option('-m', '--max-results', default=10,
-                      help='Maximum number of results to fetch')
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help='Be more verbose about errors')
-    return parser
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    try:
-        results = search(opts.title, opts.author, opts.publisher, opts.isbn,
-                         verbose=opts.verbose, max_results=int(opts.max_results))
-    except AssertionError:
-        report(True)
-        parser.print_help()
-        return 1
-    for result in results:
-        print unicode(result).encode(preferred_encoding)
-        print
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/fetch.py

@@ -1,523 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import traceback, sys, textwrap, re
-from threading import Thread
-
-from calibre import prints
-from calibre.utils.config import OptionParser
-from calibre.utils.logging import default_log
-from calibre.utils.titlecase import titlecase
-from calibre.customize import Plugin
-from calibre.ebooks.metadata.covers import check_for_cover
-from calibre.utils.html2text import html2text
-
-metadata_config = None
-
-class MetadataSource(Plugin): # {{{
-    '''
-    Represents a source to query for metadata. Subclasses must implement
-    at least the fetch method.
-
-    When :meth:`fetch` is called, the `self` object will have the following
-    useful attributes (each of which may be None)::
-
-        title, book_author, publisher, isbn, log, verbose and extra
-
-    Use these attributes to construct the search query. extra is reserved for
-    future use.
-
-    The fetch method must store the results in `self.results` as a list of
-    :class:`Metadata` objects. If there is an error, it should be stored
-    in `self.exception` and `self.tb` (for the traceback).
-    '''
-
-    author = 'Kovid Goyal'
-
-    supported_platforms = ['windows', 'osx', 'linux']
-
-    #: The type of metadata fetched. 'basic' means basic metadata like
-    #: title/author/isbn/etc. 'social' means social metadata like
-    #: tags/rating/reviews/etc.
-    metadata_type = 'basic'
-
-    #: If not None, the customization dialog will allow for string
-    #: based customization as well the default customization. The
-    #: string customization will be saved in the site_customization
-    #: member.
-    string_customization_help = None
-
-    #: Set this to true if your plugin returns HTML markup in comments.
-    #: Then if the user disables HTML, calibre will automagically convert
-    #: the HTML to Markdown.
-    has_html_comments = False
-
-    type = _('Metadata download')
-
-    def __call__(self, title, author, publisher, isbn, verbose, log=None,
-            extra=None):
-        self.worker = Thread(target=self._fetch)
-        self.worker.daemon = True
-        self.title = title
-        self.verbose = verbose
-        self.book_author = author
-        self.publisher = publisher
-        self.isbn = isbn
-        self.log = log if log is not None else default_log
-        self.extra = extra
-        self.exception, self.tb, self.results = None, None, []
-        self.worker.start()
-
-    def _fetch(self):
-        try:
-            self.fetch()
-            if self.results:
-                c = self.config_store().get(self.name, {})
-                res = self.results
-                if hasattr(res, 'authors'):
-                    res = [res]
-                for mi in res:
-                    if not c.get('rating', True):
-                        mi.rating = None
-                    if not c.get('comments', True):
-                        mi.comments = None
-                    if not c.get('tags', True):
-                        mi.tags = []
-                    if self.has_html_comments and mi.comments and \
-                            c.get('textcomments', False):
-                        try:
-                            mi.comments = html2text(mi.comments)
-                        except:
-                            traceback.print_exc()
-                            mi.comments = None
-
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    def fetch(self):
-        '''
-        All the actual work is done here.
-        '''
-        raise NotImplementedError
-
-    def join(self):
-        return self.worker.join()
-
-    def is_alive(self):
-        return self.worker.is_alive()
-
-    def is_customizable(self):
-        return True
-
-    def config_store(self):
-        global metadata_config
-        if metadata_config is None:
-            from calibre.utils.config import XMLConfig
-            metadata_config = XMLConfig('plugins/metadata_download')
-        return metadata_config
-
-    def config_widget(self):
-        from PyQt4.Qt import QWidget, QVBoxLayout, QLabel, Qt, QLineEdit, \
-            QCheckBox
-        from calibre.customize.ui import config
-        w = QWidget()
-        w._layout = QVBoxLayout(w)
-        w.setLayout(w._layout)
-        if self.string_customization_help is not None:
-            w._sc_label = QLabel(self.string_customization_help, w)
-            w._layout.addWidget(w._sc_label)
-            customization = config['plugin_customization']
-            def_sc = customization.get(self.name, '')
-            if not def_sc:
-                def_sc = ''
-            w._sc = QLineEdit(def_sc, w)
-            w._layout.addWidget(w._sc)
-            w._sc_label.setWordWrap(True)
-            w._sc_label.setTextInteractionFlags(Qt.LinksAccessibleByMouse
-                    | Qt.LinksAccessibleByKeyboard)
-            w._sc_label.setOpenExternalLinks(True)
-        c = self.config_store()
-        c = c.get(self.name, {})
-        for x, l in {'rating':_('ratings'), 'tags':_('tags'),
-                'comments':_('description/reviews')}.items():
-            cb = QCheckBox(_('Download %s from %s')%(l,
-                self.name))
-            setattr(w, '_'+x, cb)
-            cb.setChecked(c.get(x, True))
-            w._layout.addWidget(cb)
-
-        if self.has_html_comments:
-            cb = QCheckBox(_('Convert comments downloaded from %s to plain text')%(self.name))
-            setattr(w, '_textcomments', cb)
-            cb.setChecked(c.get('textcomments', False))
-            w._layout.addWidget(cb)
-
-        return w
-
-    def save_settings(self, w):
-        dl_settings = {}
-        for x in ('rating', 'tags', 'comments'):
-            dl_settings[x] = getattr(w, '_'+x).isChecked()
-        if self.has_html_comments:
-            dl_settings['textcomments'] = getattr(w, '_textcomments').isChecked()
-        c = self.config_store()
-        c.set(self.name, dl_settings)
-        if hasattr(w, '_sc'):
-            sc = unicode(w._sc.text()).strip()
-            from calibre.customize.ui import customize_plugin
-            customize_plugin(self, sc)
-
-    def customization_help(self):
-        return 'This plugin can only be customized using the GUI'
-
-    # }}}
-
-class GoogleBooks(MetadataSource): # {{{
-
-    name = 'Google Books'
-    description = _('Downloads metadata from Google Books')
-
-    def fetch(self):
-        from calibre.ebooks.metadata.google_books import search
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10,
-                                  verbose=self.verbose)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    # }}}
-
-class ISBNDB(MetadataSource): # {{{
-
-    name = 'IsbnDB'
-    description = _('Downloads metadata from isbndb.com')
-
-    def fetch(self):
-        if not self.site_customization:
-            return
-        from calibre.ebooks.metadata.isbndb import option_parser, create_books
-        args = ['isbndb']
-        if self.isbn:
-            args.extend(['--isbn', self.isbn])
-        else:
-            if self.title:
-                args.extend(['--title', self.title])
-            if self.book_author:
-                args.extend(['--author', self.book_author])
-            if self.publisher:
-                args.extend(['--publisher', self.publisher])
-        if self.verbose:
-            args.extend(['--verbose'])
-        args.append(self.site_customization) # IsbnDb key
-        try:
-            opts, args = option_parser().parse_args(args)
-            self.results = create_books(opts, args)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    @property
-    def string_customization_help(self):
-        ans = _('To use isbndb.com you must sign up for a %sfree account%s '
-                'and enter your access key below.')
-        return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
-
-    # }}}
-
-class Amazon(MetadataSource): # {{{
-
-    name = 'Amazon'
-    metadata_type = 'social'
-    description = _('Downloads social metadata from amazon.com')
-
-    has_html_comments = True
-
-    def fetch(self):
-        if not self.isbn:
-            return
-        from calibre.ebooks.metadata.amazon import get_social_metadata
-        try:
-            self.results = get_social_metadata(self.title, self.book_author,
-                    self.publisher, self.isbn)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    # }}}
-
-class KentDistrictLibrary(MetadataSource): # {{{
-
-    name = 'Kent District Library'
-    metadata_type = 'social'
-    description = _('Downloads series information from ww2.kdl.org. '
-            'This website cannot handle large numbers of queries, '
-            'so the plugin is disabled by default.')
-
-    def fetch(self):
-        if not self.title or not self.book_author:
-            return
-        from calibre.ebooks.metadata.kdl import get_series
-        try:
-            self.results = get_series(self.title, self.book_author)
-        except Exception as e:
-            import traceback
-            traceback.print_exc()
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    # }}}
-
-
-def result_index(source, result):
-    if not result.isbn:
-        return -1
-    for i, x in enumerate(source):
-        if x.isbn == result.isbn:
-            return i
-    return -1
-
-def merge_results(one, two):
-    if two is not None and one is not None:
-        for x in two:
-            idx = result_index(one, x)
-            if idx < 0:
-                one.append(x)
-            else:
-                one[idx].smart_update(x)
-
-class MetadataSources(object):
-
-    def __init__(self, sources):
-        self.sources = sources
-
-    def __enter__(self):
-        for s in self.sources:
-            s.__enter__()
-        return self
-
-    def __exit__(self, *args):
-        for s in self.sources:
-            s.__exit__()
-
-    def __call__(self, *args, **kwargs):
-        for s in self.sources:
-            s(*args, **kwargs)
-
-    def join(self):
-        for s in self.sources:
-            s.join()
-
-def filter_metadata_results(item):
-    keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
-    for keyword in keywords:
-        if item.publisher and keyword in item.publisher.lower():
-            return False
-    return True
-
-def do_cover_check(item):
-    item.has_cover = False
-    try:
-        item.has_cover = check_for_cover(item)
-    except:
-        pass # Cover not found
-
-def check_for_covers(items):
-    threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
-    for t in threads: t.start()
-    for t in threads: t.join()
-
-def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
-           verbose=0):
-    assert not(title is None and author is None and publisher is None and \
-                   isbn is None)
-    from calibre.customize.ui import metadata_sources, migrate_isbndb_key
-    migrate_isbndb_key()
-    if isbn is not None:
-        isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper()
-    fetchers = list(metadata_sources(isbndb_key=isbndb_key))
-    with MetadataSources(fetchers) as manager:
-        manager(title, author, publisher, isbn, verbose)
-        manager.join()
-
-    results = list(fetchers[0].results) if fetchers else []
-    for fetcher in fetchers[1:]:
-        merge_results(results, fetcher.results)
-
-    results = list(filter(filter_metadata_results, results))
-
-    check_for_covers(results)
-
-    words = ("the", "a", "an", "of", "and")
-    prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
-    trailing_paren_pat = re.compile(r'\(.*\)$')
-    whitespace_pat = re.compile(r'\s+')
-
-    def sort_func(x, y):
-
-        def cleanup_title(s):
-            if s is None:
-                s = _('Unknown')
-            s = s.strip().lower()
-            s = prefix_pat.sub(' ', s)
-            s = trailing_paren_pat.sub('', s)
-            s = whitespace_pat.sub(' ', s)
-            return s.strip()
-
-        t = cleanup_title(title)
-        x_title = cleanup_title(x.title)
-        y_title = cleanup_title(y.title)
-
-        # prefer titles that start with the search title
-        tx = cmp(t, x_title)
-        ty = cmp(t, y_title)
-        result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
-
-        # then prefer titles that have a cover image
-        if result == 0:
-            result = -cmp(x.has_cover, y.has_cover)
-
-        # then prefer titles with the longest comment, with in 10%
-        if result == 0:
-            cx = len(x.comments.strip() if x.comments else '')
-            cy = len(y.comments.strip() if y.comments else '')
-            t = (cx + cy) / 20
-            result = cy - cx
-            if abs(result) < t:
-                result = 0
-
-        return result
-
-    results = sorted(results, cmp=sort_func)
-
-    # if for some reason there is no comment in the top selection, go looking for one
-    if len(results) > 1:
-        if not results[0].comments or len(results[0].comments) == 0:
-            for r in results[1:]:
-                try:
-                    if title and title.lower() == r.title[:len(title)].lower() \
-                            and r.comments and len(r.comments):
-                        results[0].comments = r.comments
-                        break
-                except:
-                    pass
-        # Find a pubdate
-        pubdate = None
-        for r in results:
-            if r.pubdate is not None:
-                pubdate = r.pubdate
-                break
-        if pubdate is not None:
-            for r in results:
-                if r.pubdate is None:
-                    r.pubdate = pubdate
-
-    def fix_case(x):
-        if x:
-            x = titlecase(x)
-        return x
-
-    for r in results:
-        r.title = fix_case(r.title)
-        if r.authors:
-            r.authors = list(map(fix_case, r.authors))
-
-    return results, [(x.name, x.exception, x.tb) for x in fetchers]
-
-def get_social_metadata(mi, verbose=0):
-    from calibre.customize.ui import metadata_sources
-    fetchers = list(metadata_sources(metadata_type='social'))
-    with MetadataSources(fetchers) as manager:
-        manager(mi.title, mi.authors, mi.publisher, mi.isbn, verbose)
-        manager.join()
-    ratings, tags, comments, series, series_index = [], set([]), set([]), None, None
-    for fetcher in fetchers:
-        if fetcher.results:
-            dmi = fetcher.results
-            if dmi.rating is not None:
-                ratings.append(dmi.rating)
-            if dmi.tags:
-                for t in dmi.tags:
-                    tags.add(t)
-            if mi.pubdate is None and dmi.pubdate is not None:
-                mi.pubdate = dmi.pubdate
-            if dmi.comments:
-                comments.add(dmi.comments)
-            if dmi.series is not None:
-                series = dmi.series
-                if dmi.series_index is not None:
-                    series_index = dmi.series_index
-    if ratings:
-        rating = sum(ratings)/float(len(ratings))
-        if mi.rating is None or mi.rating < 0.1:
-            mi.rating = rating
-        else:
-            mi.rating = (mi.rating + rating)/2.0
-    if tags:
-        if not mi.tags:
-            mi.tags = []
-        mi.tags += list(tags)
-        mi.tags = list(sorted(list(set(mi.tags))))
-    if comments:
-        if not mi.comments or len(mi.comments)+20 < len(' '.join(comments)):
-            mi.comments = ''
-            for x in comments:
-                mi.comments += x+'\n\n'
-    if series and series_index is not None:
-        mi.series = series
-        mi.series_index = series_index
-
-    return [(x.name, x.exception, x.tb) for x in fetchers if x.exception is not
-            None]
-
-
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(
-        '''\
-        %prog [options]
-
-        Fetch book metadata from online sources. You must specify at least one
-        of title, author, publisher or ISBN. If you specify ISBN, the others
-        are ignored.
-        '''
-    ))
-    parser.add_option('-t', '--title', help='Book title')
-    parser.add_option('-a', '--author', help='Book author(s)')
-    parser.add_option('-p', '--publisher', help='Book publisher')
-    parser.add_option('-i', '--isbn', help='Book ISBN')
-    parser.add_option('-m', '--max-results', default=10,
-                      help='Maximum number of results to fetch')
-    parser.add_option('-k', '--isbndb-key',
-                      help=('The access key for your ISBNDB.com account. '
-                      'Only needed if you want to search isbndb.com '
-                      'and you haven\'t customized the IsbnDB plugin.'))
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help='Be more verbose about errors')
-    return parser
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    results, exceptions = search(opts.title, opts.author, opts.publisher,
-                                 opts.isbn, opts.isbndb_key, opts.verbose)
-    social_exceptions = []
-    for result in results:
-        social_exceptions.extend(get_social_metadata(result, opts.verbose))
-        prints(unicode(result))
-        print
-
-    for name, exception, tb in exceptions+social_exceptions:
-        if exception is not None:
-            print 'WARNING: Fetching from', name, 'failed with error:'
-            print exception
-            print tb
-
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/fictionwise.py

@@ -1,390 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2010, sengian <sengian1@gmail.com>'
-__docformat__ = 'restructuredtext en'
-
-import sys, textwrap, re, traceback, socket
-from urllib import urlencode
-
-from lxml.html import soupparser, tostring
-
-from calibre import browser, preferred_encoding
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.ebooks.metadata import MetaInformation, check_isbn, \
-    authors_to_sort_string
-from calibre.library.comments import sanitize_comments_html
-from calibre.ebooks.metadata.fetch import MetadataSource
-from calibre.utils.config import OptionParser
-from calibre.utils.date import parse_date, utcnow
-from calibre.utils.cleantext import clean_ascii_chars
-
-class Fictionwise(MetadataSource): # {{{
-
-    author = 'Sengian'
-    name = 'Fictionwise'
-    description = _('Downloads metadata from Fictionwise')
-
-    has_html_comments = True
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                self.isbn, max_results=10, verbose=self.verbose)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    # }}}
-
-class FictionwiseError(Exception):
-    pass
-
-def report(verbose):
-    if verbose:
-        traceback.print_exc()
-
-class Query(object):
-
-    BASE_URL = 'http://www.fictionwise.com/servlet/mw'
-
-    def __init__(self, title=None, author=None, publisher=None, keywords=None, max_results=20):
-        assert not(title is None and author is None and publisher is None and keywords is None)
-        assert (max_results < 21)
-
-        self.max_results = int(max_results)
-        q = {   'template' : 'searchresults_adv.htm' ,
-                'searchtitle' : '',
-                'searchauthor' : '',
-                'searchpublisher' : '',
-                'searchkeyword' : '',
-                #possibilities startoflast, fullname, lastfirst
-                'searchauthortype' : 'startoflast',
-                'searchcategory' : '',
-                'searchcategory2' : '',
-                'searchprice_s' : '0',
-                'searchprice_e' : 'ANY',
-                'searchformat' : '',
-                'searchgeo' : 'US',
-                'searchfwdatetype' : '',
-                #maybe use dates fields if needed?
-                #'sortorder' : 'DESC',
-                #many options available: b.SortTitle, a.SortName,
-                #b.DateFirstPublished, b.FWPublishDate
-                'sortby' : 'b.SortTitle'
-            }
-        if title is not None:
-            q['searchtitle'] = title
-        if author is not None:
-            q['searchauthor'] = author
-        if publisher is not None:
-            q['searchpublisher'] = publisher
-        if keywords is not None:
-            q['searchkeyword'] = keywords
-
-        if isinstance(q, unicode):
-            q = q.encode('utf-8')
-        self.urldata = urlencode(q)
-
-    def __call__(self, browser, verbose, timeout = 5.):
-        if verbose:
-            print _('Query: %s') % self.BASE_URL+self.urldata
-
-        try:
-            raw = browser.open_novisit(self.BASE_URL, self.urldata, timeout=timeout).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
-                raise FictionwiseError(_('Fictionwise timed out. Try again later.'))
-            raise FictionwiseError(_('Fictionwise encountered an error.'))
-        if '<title>404 - ' in raw:
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-        try:
-            feed = soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                feed = soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                return None
-
-        # get list of results as links
-        results = feed.xpath("//table[3]/tr/td[2]/table/tr/td/p/table[2]/tr[@valign]")
-        results = results[:self.max_results]
-        results = [i.xpath('descendant-or-self::a')[0].get('href') for i in results]
-        #return feed if no links ie normally a single book or nothing
-        if not results:
-            results = [feed]
-        return results
-
-class ResultList(list):
-
-    BASE_URL = 'http://www.fictionwise.com'
-    COLOR_VALUES = {'BLUE': 4, 'GREEN': 3, 'YELLOW': 2, 'RED': 1, 'NA': 0}
-
-    def __init__(self):
-        self.retitle = re.compile(r'\[[^\[\]]+\]')
-        self.rechkauth = re.compile(r'.*book\s*by', re.I)
-        self.redesc = re.compile(r'book\s*description\s*:\s*(<br[^>]+>)*(?P<desc>.*)<br[^>]*>.{,15}publisher\s*:', re.I)
-        self.repub = re.compile(r'.*publisher\s*:\s*', re.I)
-        self.redate = re.compile(r'.*release\s*date\s*:\s*', re.I)
-        self.retag = re.compile(r'.*book\s*category\s*:\s*', re.I)
-        self.resplitbr = re.compile(r'<br[^>]*>', re.I)
-        self.recomment = re.compile(r'(?s)<!--.*?-->')
-        self.reimg = re.compile(r'<img[^>]*>', re.I)
-        self.resanitize = re.compile(r'\[HTML_REMOVED\]\s*', re.I)
-        self.renbcom = re.compile('(?P<nbcom>\d+)\s*Reader Ratings:')
-        self.recolor = re.compile('(?P<ncolor>[^/]+).gif')
-        self.resplitbrdiv = re.compile(r'(<br[^>]+>|</?div[^>]*>)', re.I)
-        self.reisbn = re.compile(r'.*ISBN\s*:\s*', re.I)
-
-    def strip_tags_etree(self, etreeobj, invalid_tags):
-        for (itag, rmv) in invalid_tags.iteritems():
-            if rmv:
-                for elts in etreeobj.getiterator(itag):
-                    elts.drop_tree()
-            else:
-                for elts in etreeobj.getiterator(itag):
-                    elts.drop_tag()
-
-    def clean_entry(self, entry, invalid_tags = {'script': True},
-                invalid_id = (), invalid_class=(), invalid_xpath = ()):
-        #invalid_tags: remove tag and keep content if False else remove
-        #remove tags
-        if invalid_tags:
-            self.strip_tags_etree(entry, invalid_tags)
-        #remove xpath
-        if invalid_xpath:
-            for eltid in invalid_xpath:
-                elt = entry.xpath(eltid)
-                for el in elt:
-                    el.drop_tree()
-        #remove id
-        if invalid_id:
-            for eltid in invalid_id:
-                elt = entry.get_element_by_id(eltid)
-                if elt is not None:
-                    elt.drop_tree()
-        #remove class
-        if invalid_class:
-            for eltclass in invalid_class:
-                elts = entry.find_class(eltclass)
-                if elts is not None:
-                    for elt in elts:
-                        elt.drop_tree()
-
-    def output_entry(self, entry, prettyout = True, htmlrm="\d+"):
-        out = tostring(entry, pretty_print=prettyout)
-        #try to work around tostring to remove this encoding for exemle
-        reclean = re.compile('(\n+|\t+|\r+|&#'+htmlrm+';)')
-        return reclean.sub('', out)
-
-    def get_title(self, entry):
-        title = entry.findtext('./')
-        return self.retitle.sub('', title).strip()
-
-    def get_authors(self, entry):
-        authortext = entry.find('./br').tail
-        if not self.rechkauth.search(authortext):
-            return []
-        authortext = self.rechkauth.sub('', authortext)
-        return [a.strip() for a in authortext.split('&')]
-
-    def get_rating(self, entrytable, verbose):
-        nbcomment = tostring(entrytable.getprevious())
-        try:
-            nbcomment = self.renbcom.search(nbcomment).group("nbcom")
-        except:
-            report(verbose)
-            return None
-        hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")],
-                    float(image.get('height', default=0))) \
-                        for image in entrytable.getiterator('img'))
-        #ratings as x/5
-        return float(1.25*sum(k*v for (k, v) in hval.iteritems())/sum(hval.itervalues()))
-
-    def get_description(self, entry):
-        description = self.output_entry(entry.xpath('./p')[1],htmlrm="")
-        description = self.redesc.search(description)
-        if not description or not description.group("desc"):
-            return None
-        #remove invalid tags
-        description = self.reimg.sub('', description.group("desc"))
-        description = self.recomment.sub('', description)
-        description = self.resanitize.sub('', sanitize_comments_html(description))
-        return _('SUMMARY:\n %s') % re.sub(r'\n\s+</p>','\n</p>', description)
-
-    def get_publisher(self, entry):
-        publisher = self.output_entry(entry.xpath('./p')[1])
-        publisher = filter(lambda x: self.repub.search(x) is not None,
-            self.resplitbr.split(publisher))
-        if not len(publisher):
-            return None
-        publisher = self.repub.sub('', publisher[0])
-        return publisher.split(',')[0].strip()
-
-    def get_tags(self, entry):
-        tag = self.output_entry(entry.xpath('./p')[1])
-        tag = filter(lambda x: self.retag.search(x) is not None,
-            self.resplitbr.split(tag))
-        if not len(tag):
-            return []
-        return map(lambda x: x.strip(), self.retag.sub('', tag[0]).split('/'))
-
-    def get_date(self, entry, verbose):
-        date = self.output_entry(entry.xpath('./p')[1])
-        date = filter(lambda x: self.redate.search(x) is not None,
-            self.resplitbr.split(date))
-        if not len(date):
-            return None
-        try:
-            d = self.redate.sub('', date[0])
-            if d:
-                default = utcnow().replace(day=15)
-                d = parse_date(d, assume_utc=True, default=default)
-            else:
-                d = None
-        except:
-            report(verbose)
-            d = None
-        return d
-
-    def get_ISBN(self, entry):
-        isbns = self.output_entry(entry.xpath('./p')[2])
-        isbns = filter(lambda x: self.reisbn.search(x) is not None,
-            self.resplitbrdiv.split(isbns))
-        if not len(isbns):
-            return None
-        isbns = [self.reisbn.sub('', x) for x in isbns if check_isbn(self.reisbn.sub('', x))]
-        return sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1]
-
-    def fill_MI(self, entry, title, authors, ratings, verbose):
-        mi = MetaInformation(title, authors)
-        mi.rating = ratings
-        mi.comments = self.get_description(entry)
-        mi.publisher = self.get_publisher(entry)
-        mi.tags = self.get_tags(entry)
-        mi.pubdate = self.get_date(entry, verbose)
-        mi.isbn = self.get_ISBN(entry)
-        mi.author_sort = authors_to_sort_string(authors)
-        return mi
-
-    def get_individual_metadata(self, browser, linkdata, verbose):
-        try:
-            raw = browser.open_novisit(self.BASE_URL + linkdata).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
-                raise FictionwiseError(_('Fictionwise timed out. Try again later.'))
-            raise FictionwiseError(_('Fictionwise encountered an error.'))
-        if '<title>404 - ' in raw:
-            report(verbose)
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-        try:
-            return soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                return soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                return None
-
-    def populate(self, entries, browser, verbose=False):
-        inv_tags ={'script': True, 'a': False, 'font': False, 'strong': False, 'b': False,
-            'ul': False, 'span': False}
-        inv_xpath =('./table',)
-        #single entry
-        if len(entries) == 1 and not isinstance(entries[0], str):
-            try:
-                entry = entries.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")
-                self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath)
-                title = self.get_title(entry)
-                #maybe strenghten the search
-                ratings =  self.get_rating(entry.xpath("./p/table")[1], verbose)
-                authors = self.get_authors(entry)
-            except Exception as e:
-                if verbose:
-                    print _('Failed to get all details for an entry')
-                    print e
-                return
-            self.append(self.fill_MI(entry, title, authors, ratings, verbose))
-        else:
-            #multiple entries
-            for x in entries:
-                try:
-                    entry = self.get_individual_metadata(browser, x, verbose)
-                    entry = entry.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")[0]
-                    self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath)
-                    title = self.get_title(entry)
-                    #maybe strenghten the search
-                    ratings =  self.get_rating(entry.xpath("./p/table")[1], verbose)
-                    authors = self.get_authors(entry)
-                except Exception as e:
-                    if verbose:
-                        print _('Failed to get all details for an entry')
-                        print e
-                    continue
-                self.append(self.fill_MI(entry, title, authors, ratings, verbose))
-
-
-def search(title=None, author=None, publisher=None, isbn=None,
-           min_viewability='none', verbose=False, max_results=5,
-            keywords=None):
-    br = browser()
-    entries = Query(title=title, author=author, publisher=publisher,
-        keywords=keywords, max_results=max_results)(br, verbose, timeout = 15.)
-
-    #List of entry
-    ans = ResultList()
-    ans.populate(entries, br, verbose)
-    return ans
-
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(\
-    _('''\
-        %prog [options]
-
-        Fetch book metadata from Fictionwise. You must specify one of title, author,
-        or keywords. No ISBN specification possible. Will fetch a maximum of 20 matches,
-        so you should make your query as specific as possible.
-    ''')
-    ))
-    parser.add_option('-t', '--title', help=_('Book title'))
-    parser.add_option('-a', '--author', help=_('Book author(s)'))
-    parser.add_option('-p', '--publisher', help=_('Book publisher'))
-    parser.add_option('-k', '--keywords', help=_('Keywords'))
-    parser.add_option('-m', '--max-results', default=20,
-                      help=_('Maximum number of results to fetch'))
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help=_('Be more verbose about errors'))
-    return parser
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    try:
-        results = search(opts.title, opts.author, publisher=opts.publisher,
-            keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results)
-    except AssertionError:
-        report(True)
-        parser.print_help()
-        return 1
-    if results is None or len(results) == 0:
-        print _('No result found for this search!')
-        return 0
-    for result in results:
-        print unicode(result).encode(preferred_encoding, 'replace')
-        print
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/google_books.py

@@ -1,247 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import sys, textwrap
-from urllib import urlencode
-from functools import partial
-
-from lxml import etree
-
-from calibre import browser, preferred_encoding
-from calibre.ebooks.metadata import MetaInformation
-from calibre.utils.config import OptionParser
-from calibre.utils.date import parse_date, utcnow
-
-NAMESPACES = {
-              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
-              'atom' : 'http://www.w3.org/2005/Atom',
-              'dc': 'http://purl.org/dc/terms'
-            }
-XPath = partial(etree.XPath, namespaces=NAMESPACES)
-
-total_results  = XPath('//openSearch:totalResults')
-start_index    = XPath('//openSearch:startIndex')
-items_per_page = XPath('//openSearch:itemsPerPage')
-entry          = XPath('//atom:entry')
-entry_id       = XPath('descendant::atom:id')
-creator        = XPath('descendant::dc:creator')
-identifier     = XPath('descendant::dc:identifier')
-title          = XPath('descendant::dc:title')
-date           = XPath('descendant::dc:date')
-publisher      = XPath('descendant::dc:publisher')
-subject        = XPath('descendant::dc:subject')
-description    = XPath('descendant::dc:description')
-language       = XPath('descendant::dc:language')
-
-def report(verbose):
-    if verbose:
-        import traceback
-        traceback.print_exc()
-
-
-class Query(object):
-
-    BASE_URL = 'http://books.google.com/books/feeds/volumes?'
-
-    def __init__(self, title=None, author=None, publisher=None, isbn=None,
-                 max_results=20, min_viewability='none', start_index=1):
-        assert not(title is None and author is None and publisher is None and \
-                   isbn is None)
-        assert (max_results < 21)
-        assert (min_viewability in ('none', 'partial', 'full'))
-        q = ''
-        if isbn is not None:
-            q += 'isbn:'+isbn
-        else:
-            def build_term(prefix, parts):
-                return ' '.join('in'+prefix + ':' + x for x in parts)
-            if title is not None:
-                q += build_term('title', title.split())
-            if author is not None:
-                q += ('+' if q else '')+build_term('author', author.split())
-            if publisher is not None:
-                q += ('+' if q else '')+build_term('publisher', publisher.split())
-
-        if isinstance(q, unicode):
-            q = q.encode('utf-8')
-        self.url = self.BASE_URL+urlencode({
-            'q':q,
-            'max-results':max_results,
-            'start-index':start_index,
-            'min-viewability':min_viewability,
-            })
-
-    def __call__(self, browser, verbose):
-        if verbose:
-            print 'Query:', self.url
-        feed = etree.fromstring(browser.open(self.url).read())
-        #print etree.tostring(feed, pretty_print=True)
-        total = int(total_results(feed)[0].text)
-        start = int(start_index(feed)[0].text)
-        entries = entry(feed)
-        new_start = start + len(entries)
-        if new_start > total:
-            new_start = 0
-        return entries, new_start
-
-
-class ResultList(list):
-
-    def get_description(self, entry, verbose):
-        try:
-            desc = description(entry)
-            if desc:
-                return 'SUMMARY:\n'+desc[0].text
-        except:
-            report(verbose)
-
-    def get_language(self, entry, verbose):
-        try:
-            l = language(entry)
-            if l:
-                return l[0].text
-        except:
-            report(verbose)
-
-    def get_title(self, entry):
-        candidates = [x.text for x in title(entry)]
-        return ': '.join(candidates)
-
-    def get_authors(self, entry):
-        m = creator(entry)
-        if not m:
-            m = []
-        m = [x.text for x in m]
-        return m
-
-    def get_author_sort(self, entry, verbose):
-        for x in creator(entry):
-            for key, val in x.attrib.items():
-                if key.endswith('file-as'):
-                    return val
-
-    def get_identifiers(self, entry, mi):
-        isbns = []
-        for x in identifier(entry):
-            t = str(x.text).strip()
-            if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'):
-                if t[:5].upper() == 'ISBN:':
-                    isbns.append(t[5:])
-        if isbns:
-            mi.isbn = sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1]
-
-    def get_tags(self, entry, verbose):
-        try:
-            btags = [x.text for x in subject(entry)]
-            tags = []
-            for t in btags:
-                tags.extend([y.strip() for y in t.split('/')])
-            tags = list(sorted(list(set(tags))))
-        except:
-            report(verbose)
-            tags = []
-        return [x.replace(',', ';') for x in tags]
-
-    def get_publisher(self, entry, verbose):
-        try:
-            pub = publisher(entry)[0].text
-        except:
-            pub = None
-        return pub
-
-    def get_date(self, entry, verbose):
-        try:
-            d = date(entry)
-            if d:
-                default = utcnow().replace(day=15)
-                d = parse_date(d[0].text, assume_utc=True, default=default)
-            else:
-                d = None
-        except:
-            report(verbose)
-            d = None
-        return d
-
-    def populate(self, entries, browser, verbose=False):
-        for x in entries:
-            try:
-                id_url = entry_id(x)[0].text
-                title = self.get_title(x)
-            except:
-                report(verbose)
-            mi = MetaInformation(title, self.get_authors(x))
-            try:
-                raw = browser.open(id_url).read()
-                feed = etree.fromstring(raw)
-                x = entry(feed)[0]
-            except Exception as e:
-                if verbose:
-                    print 'Failed to get all details for an entry'
-                    print e
-            mi.author_sort = self.get_author_sort(x, verbose)
-            mi.comments = self.get_description(x, verbose)
-            self.get_identifiers(x, mi)
-            mi.tags = self.get_tags(x, verbose)
-            mi.publisher = self.get_publisher(x, verbose)
-            mi.pubdate = self.get_date(x, verbose)
-            mi.language = self.get_language(x, verbose)
-            self.append(mi)
-
-
-def search(title=None, author=None, publisher=None, isbn=None,
-           min_viewability='none', verbose=False, max_results=40):
-    br   = browser()
-    br.set_handle_gzip(True)
-    start, entries = 1, []
-    while start > 0 and len(entries) <= max_results:
-        new, start = Query(title=title, author=author, publisher=publisher,
-                       isbn=isbn, min_viewability=min_viewability)(br, verbose)
-        if not new:
-            break
-        entries.extend(new)
-
-    entries = entries[:max_results]
-
-    ans = ResultList()
-    ans.populate(entries, br, verbose)
-    return ans
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(
-        '''\
-        %prog [options]
-
-        Fetch book metadata from Google. You must specify one of title, author,
-        publisher or ISBN. If you specify ISBN the others are ignored. Will
-        fetch a maximum of 100 matches, so you should make your query as
-        specific as possible.
-        '''
-    ))
-    parser.add_option('-t', '--title', help='Book title')
-    parser.add_option('-a', '--author', help='Book author(s)')
-    parser.add_option('-p', '--publisher', help='Book publisher')
-    parser.add_option('-i', '--isbn', help='Book ISBN')
-    parser.add_option('-m', '--max-results', default=10,
-                      help='Maximum number of results to fetch')
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help='Be more verbose about errors')
-    return parser
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    try:
-        results = search(opts.title, opts.author, opts.publisher, opts.isbn,
-                         verbose=opts.verbose, max_results=opts.max_results)
-    except AssertionError:
-        report(True)
-        parser.print_help()
-        return 1
-    for result in results:
-        print unicode(result).encode(preferred_encoding)
-        print
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/isbndb.py

@@ -1,159 +0,0 @@
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-'''
-Interface to isbndb.com. My key HLLXQX2A.
-'''
-
-import sys, re
-from urllib import quote
-
-from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata.book.base import Metadata
-from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
-from calibre import browser
-
-BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&results=subjects,authors,texts&'
-
-class ISBNDBError(Exception):
-    pass
-
-def fetch_metadata(url, max=3, timeout=5.):
-    books = []
-    page_number = 1
-    total_results = 31
-    br = browser()
-    while len(books) < total_results and max > 0:
-        try:
-            raw = br.open(url, timeout=timeout).read()
-        except Exception as err:
-            raise ISBNDBError('Could not fetch ISBNDB metadata. Error: '+str(err))
-        soup = BeautifulStoneSoup(raw,
-                convertEntities=BeautifulStoneSoup.XML_ENTITIES)
-        book_list = soup.find('booklist')
-        if book_list is None:
-            errmsg = soup.find('errormessage').string
-            raise ISBNDBError('Error fetching metadata: '+errmsg)
-        total_results = int(book_list['total_results'])
-        page_number += 1
-        np = '&page_number=%s&'%page_number
-        url = re.sub(r'\&page_number=\d+\&', np, url)
-        books.extend(book_list.findAll('bookdata'))
-        max -= 1
-    return books
-
-
-class ISBNDBMetadata(Metadata):
-
-    def __init__(self, book):
-        Metadata.__init__(self, None)
-
-        def tostring(e):
-            if not hasattr(e, 'string'):
-                return None
-            ans = e.string
-            if ans is not None:
-                ans = unicode(ans).strip()
-            if not ans:
-                ans = None
-            return ans
-
-        self.isbn = unicode(book.get('isbn13', book.get('isbn')))
-        title = tostring(book.find('titlelong'))
-        if not title:
-            title = tostring(book.find('title'))
-        self.title = title
-        self.title = unicode(self.title).strip()
-        authors = []
-        au = tostring(book.find('authorstext'))
-        if au:
-            au = au.strip()
-            temp = au.split(',')
-            for au in temp:
-                if not au: continue
-                authors.extend([a.strip() for a in au.split('&amp;')])
-        if authors:
-            self.authors = authors
-        try:
-            self.author_sort = tostring(book.find('authors').find('person'))
-            if self.authors and self.author_sort == self.authors[0]:
-                self.author_sort = None
-        except:
-            pass
-        self.publisher = tostring(book.find('publishertext'))
-
-        summ = tostring(book.find('summary'))
-        if summ:
-            self.comments = 'SUMMARY:\n'+summ
-
-
-def build_isbn(base_url, opts):
-    return base_url + 'index1=isbn&value1='+opts.isbn
-
-def build_combined(base_url, opts):
-    query = ' '.join([e for e in (opts.title, opts.author, opts.publisher) \
-        if e is not None ])
-    query = query.strip()
-    if len(query) == 0:
-        raise ISBNDBError('You must specify at least one of --author, --title or --publisher')
-
-    query = re.sub(r'\s+', '+', query)
-    if isinstance(query, unicode):
-        query = query.encode('utf-8')
-    return base_url+'index1=combined&value1='+quote(query, '+')
-
-
-def option_parser():
-    parser = OptionParser(usage=\
-_('''
-%prog [options] key
-
-Fetch metadata for books from isndb.com. You can specify either the
-books ISBN ID or its title and author. If you specify the title and author,
-then more than one book may be returned.
-
-key is the account key you generate after signing up for a free account from isbndb.com.
-
-'''))
-    parser.add_option('-i', '--isbn', default=None, dest='isbn',
-                      help=_('The ISBN ID of the book you want metadata for.'))
-    parser.add_option('-a', '--author', dest='author',
-                      default=None, help=_('The author whose book to search for.'))
-    parser.add_option('-t', '--title', dest='title',
-                      default=None, help=_('The title of the book to search for.'))
-    parser.add_option('-p', '--publisher', default=None, dest='publisher',
-                      help=_('The publisher of the book to search for.'))
-    parser.add_option('-v', '--verbose', default=False,
-                      action='store_true', help=_('Verbose processing'))
-
-    return parser
-
-
-def create_books(opts, args, timeout=5.):
-    base_url = BASE_URL%dict(key=args[1])
-    if opts.isbn is not None:
-        url = build_isbn(base_url, opts)
-    else:
-        url = build_combined(base_url, opts)
-
-    if opts.verbose:
-        print ('ISBNDB query: '+url)
-
-    tans = [ISBNDBMetadata(book) for book in fetch_metadata(url, timeout=timeout)]
-    #remove duplicates ISBN
-    return list(dict((book.isbn, book) for book in tans).values())
-
-def main(args=sys.argv):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    if len(args) != 2:
-        parser.print_help()
-        print ('You must supply the isbndb.com key')
-        return 1
-
-    for book in create_books(opts, args):
-        print unicode(book).encode('utf-8')
-
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/nicebooks.py

@@ -1,411 +0,0 @@
-from __future__ import with_statement
-__license__ = 'GPL 3'
-__copyright__ = '2010, sengian <sengian1@gmail.com>'
-__docformat__ = 'restructuredtext en'
-
-import sys, textwrap, re, traceback, socket
-from urllib import urlencode
-from math import ceil
-from copy import deepcopy
-
-from lxml.html import soupparser
-
-from calibre.utils.date import parse_date, utcnow, replace_months
-from calibre.utils.cleantext import clean_ascii_chars
-from calibre import browser, preferred_encoding
-from calibre.ebooks.chardet import xml_to_unicode
-from calibre.ebooks.metadata import MetaInformation, check_isbn, \
-    authors_to_sort_string
-from calibre.ebooks.metadata.fetch import MetadataSource
-from calibre.ebooks.metadata.covers import CoverDownload
-from calibre.utils.config import OptionParser
-
-class NiceBooks(MetadataSource):
-
-    name = 'Nicebooks'
-    description = _('Downloads metadata from french Nicebooks')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    version             = (1, 0, 0)
-
-    def fetch(self):
-        try:
-            self.results = search(self.title, self.book_author, self.publisher,
-                                  self.isbn, max_results=10, verbose=self.verbose)
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-class NiceBooksCovers(CoverDownload):
-
-    name = 'Nicebooks covers'
-    description = _('Downloads covers from french Nicebooks')
-    supported_platforms = ['windows', 'osx', 'linux']
-    author = 'Sengian'
-    type = _('Cover download')
-    version             = (1, 0, 0)
-
-    def has_cover(self, mi, ans, timeout=5.):
-        if not mi.isbn:
-            return False
-        br = browser()
-        try:
-            entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0]
-            if Covers(mi.isbn)(entry).check_cover():
-                self.debug('cover for', mi.isbn, 'found')
-                ans.set()
-        except Exception as e:
-            self.debug(e)
-
-    def get_covers(self, mi, result_queue, abort, timeout=5.):
-        if not mi.isbn:
-            return
-        br = browser()
-        try:
-            entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0]
-            cover_data, ext = Covers(mi.isbn)(entry).get_cover(br, timeout)
-            if not ext:
-                ext = 'jpg'
-            result_queue.put((True, cover_data, ext, self.name))
-        except Exception as e:
-            result_queue.put((False, self.exception_to_string(e),
-                traceback.format_exc(), self.name))
-
-
-class NiceBooksError(Exception):
-    pass
-
-class ISBNNotFound(NiceBooksError):
-    pass
-
-def report(verbose):
-    if verbose:
-        traceback.print_exc()
-
-class Query(object):
-
-    BASE_URL = 'http://fr.nicebooks.com/'
-
-    def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, max_results=20):
-        assert not(title is None and author is None and publisher is None \
-            and isbn is None and keywords is None)
-        assert (max_results < 21)
-
-        self.max_results = int(max_results)
-
-        if isbn is not None:
-            q = isbn
-        else:
-            q = ' '.join([i for i in (title, author, publisher, keywords) \
-                if i is not None])
-
-        if isinstance(q, unicode):
-            q = q.encode('utf-8')
-        self.urldata = 'search?' + urlencode({'q':q,'s':'Rechercher'})
-
-    def __call__(self, browser, verbose, timeout = 5.):
-        if verbose:
-            print _('Query: %s') % self.BASE_URL+self.urldata
-
-        try:
-            raw = browser.open_novisit(self.BASE_URL+self.urldata, timeout=timeout).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
-                raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
-            raise NiceBooksError(_('Nicebooks encountered an error.'))
-        if '<title>404 - ' in raw:
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-        try:
-            feed = soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                feed = soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                return None
-
-        #nb of page to call
-        try:
-            nbresults = int(feed.xpath("//div[@id='topbar']/b")[0].text)
-        except:
-            #direct hit
-            return [feed]
-
-        nbpagetoquery = int(ceil(float(min(nbresults, self.max_results))/10))
-        pages =[feed]
-        if nbpagetoquery > 1:
-            for i in xrange(2, nbpagetoquery + 1):
-                try:
-                    urldata = self.urldata + '&p=' + str(i)
-                    raw = browser.open_novisit(self.BASE_URL+urldata, timeout=timeout).read()
-                except Exception as e:
-                    continue
-                if '<title>404 - ' in raw:
-                    continue
-                raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                        resolve_entities=True)[0]
-                try:
-                    feed = soupparser.fromstring(raw)
-                except:
-                    try:
-                        #remove ASCII invalid chars
-                        feed = soupparser.fromstring(clean_ascii_chars(raw))
-                    except:
-                        continue
-                pages.append(feed)
-
-        results = []
-        for x in pages:
-            results.extend([i.find_class('title')[0].get('href') \
-                for i in x.xpath("//ul[@id='results']/li")])
-        return results[:self.max_results]
-
-class ResultList(list):
-
-    BASE_URL = 'http://fr.nicebooks.com'
-
-    def __init__(self):
-        self.repub = re.compile(u'\s*.diteur\s*', re.I)
-        self.reauteur = re.compile(u'\s*auteur.*', re.I)
-        self.reautclean = re.compile(u'\s*\(.*\)\s*')
-
-    def get_title(self, entry):
-        title = deepcopy(entry)
-        title.remove(title.find("dl[@title='Informations sur le livre']"))
-        title = ' '.join([i.text_content() for i in title.iterchildren()])
-        return unicode(title.replace('\n', ''))
-
-    def get_authors(self, entry):
-        author = entry.find("dl[@title='Informations sur le livre']")
-        authortext = []
-        for x in author.getiterator('dt'):
-            if self.reauteur.match(x.text):
-                elt = x.getnext()
-                while elt.tag == 'dd':
-                    authortext.append(unicode(elt.text_content()))
-                    elt = elt.getnext()
-                break
-        if len(authortext) == 1:
-            authortext = [self.reautclean.sub('', authortext[0])]
-        return authortext
-
-    def get_description(self, entry, verbose):
-        try:
-            return u'RESUME:\n' + unicode(entry.getparent().xpath("//p[@id='book-description']")[0].text)
-        except:
-            report(verbose)
-            return None
-
-    def get_book_info(self, entry, mi, verbose):
-        entry = entry.find("dl[@title='Informations sur le livre']")
-        for x in entry.getiterator('dt'):
-            if x.text == 'ISBN':
-                isbntext = x.getnext().text_content().replace('-', '')
-                if check_isbn(isbntext):
-                    mi.isbn = unicode(isbntext)
-            elif self.repub.match(x.text):
-                mi.publisher = unicode(x.getnext().text_content())
-            elif x.text == 'Langue':
-                mi.language = unicode(x.getnext().text_content())
-            elif x.text == 'Date de parution':
-                d = x.getnext().text_content()
-                try:
-                    default = utcnow().replace(day=15)
-                    d = replace_months(d, 'fr')
-                    d = parse_date(d, assume_utc=True, default=default)
-                    mi.pubdate = d
-                except:
-                    report(verbose)
-        return mi
-
-    def fill_MI(self, entry, title, authors, verbose):
-        mi = MetaInformation(title, authors)
-        mi.author_sort = authors_to_sort_string(authors)
-        mi.comments = self.get_description(entry, verbose)
-        return self.get_book_info(entry, mi, verbose)
-
-    def get_individual_metadata(self, browser, linkdata, verbose):
-        try:
-            raw = browser.open_novisit(self.BASE_URL + linkdata).read()
-        except Exception as e:
-            report(verbose)
-            if callable(getattr(e, 'getcode', None)) and \
-                    e.getcode() == 404:
-                return
-            if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
-                raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
-            raise NiceBooksError(_('Nicebooks encountered an error.'))
-        if '<title>404 - ' in raw:
-            report(verbose)
-            return
-        raw = xml_to_unicode(raw, strip_encoding_pats=True,
-                resolve_entities=True)[0]
-        try:
-            feed = soupparser.fromstring(raw)
-        except:
-            try:
-                #remove ASCII invalid chars
-                feed = soupparser.fromstring(clean_ascii_chars(raw))
-            except:
-                return None
-
-        # get results
-        return feed.xpath("//div[@id='container']")[0]
-
-    def populate(self, entries, browser, verbose=False):
-        #single entry
-        if len(entries) == 1 and not isinstance(entries[0], str):
-            try:
-                entry = entries[0].xpath("//div[@id='container']")[0]
-                entry = entry.find("div[@id='book-info']")
-                title = self.get_title(entry)
-                authors = self.get_authors(entry)
-            except Exception as e:
-                if verbose:
-                    print 'Failed to get all details for an entry'
-                    print e
-                return
-            self.append(self.fill_MI(entry, title, authors, verbose))
-        else:
-        #multiple entries
-            for x in entries:
-                try:
-                    entry = self.get_individual_metadata(browser, x, verbose)
-                    entry = entry.find("div[@id='book-info']")
-                    title = self.get_title(entry)
-                    authors = self.get_authors(entry)
-                except Exception as e:
-                    if verbose:
-                        print 'Failed to get all details for an entry'
-                        print e
-                    continue
-                self.append(self.fill_MI(entry, title, authors, verbose))
-
-class Covers(object):
-
-    def __init__(self, isbn = None):
-        assert isbn is not None
-        self.urlimg = ''
-        self.isbn = isbn
-        self.isbnf = False
-
-    def __call__(self, entry = None):
-        try:
-            self.urlimg = entry.xpath("//div[@id='book-picture']/a")[0].get('href')
-        except:
-            return self
-        isbno = entry.get_element_by_id('book-info').find("dl[@title='Informations sur le livre']")
-        for x in isbno.getiterator('dt'):
-            if x.text == 'ISBN' and check_isbn(x.getnext().text_content()):
-                self.isbnf = True
-                break
-        return self
-
-    def check_cover(self):
-        return True if self.urlimg else False
-
-    def get_cover(self, browser, timeout = 5.):
-        try:
-            cover, ext = browser.open_novisit(self.urlimg, timeout=timeout).read(), \
-                self.urlimg.rpartition('.')[-1]
-            return cover, ext if ext else 'jpg'
-        except Exception as err:
-            if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-                raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
-            if not len(self.urlimg):
-                if not self.isbnf:
-                    raise ISBNNotFound(_('ISBN: %s not found.') % self.isbn)
-                raise NiceBooksError(_('An errror occured with Nicebooks cover fetcher'))
-
-
-def search(title=None, author=None, publisher=None, isbn=None,
-           max_results=5, verbose=False, keywords=None):
-    br = browser()
-    entries = Query(title=title, author=author, isbn=isbn, publisher=publisher,
-        keywords=keywords, max_results=max_results)(br, verbose,timeout = 10.)
-
-    if entries is None or len(entries) == 0:
-        return None
-
-    #List of entry
-    ans = ResultList()
-    ans.populate(entries, br, verbose)
-    return ans
-
-def check_for_cover(isbn):
-    br = browser()
-    entry = Query(isbn=isbn, max_results=1)(br, False)[0]
-    return Covers(isbn)(entry).check_cover()
-
-def cover_from_isbn(isbn, timeout = 5.):
-    br = browser()
-    entry = Query(isbn=isbn, max_results=1)(br, False, timeout)[0]
-    return Covers(isbn)(entry).get_cover(br, timeout)
-
-
-def option_parser():
-    parser = OptionParser(textwrap.dedent(\
-    _('''\
-        %prog [options]
-
-        Fetch book metadata from Nicebooks. You must specify one of title, author,
-        ISBN, publisher or keywords. Will fetch a maximum of 20 matches,
-        so you should make your query as specific as possible.
-        It can also get covers if the option is activated.
-    ''')
-    ))
-    parser.add_option('-t', '--title', help=_('Book title'))
-    parser.add_option('-a', '--author', help=_('Book author(s)'))
-    parser.add_option('-p', '--publisher', help=_('Book publisher'))
-    parser.add_option('-i', '--isbn', help=_('Book ISBN'))
-    parser.add_option('-k', '--keywords', help=_('Keywords'))
-    parser.add_option('-c', '--covers', default=0,
-                      help=_('Covers: 1-Check/ 2-Download'))
-    parser.add_option('-p', '--coverspath', default='',
-                      help=_('Covers files path'))
-    parser.add_option('-m', '--max-results', default=20,
-                      help=_('Maximum number of results to fetch'))
-    parser.add_option('-v', '--verbose', default=0, action='count',
-                      help=_('Be more verbose about errors'))
-    return parser
-
-def main(args=sys.argv):
-    import os
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    try:
-        results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher,
-            keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results)
-    except AssertionError:
-        report(True)
-        parser.print_help()
-        return 1
-    if results is None or len(results) == 0:
-        print _('No result found for this search!')
-        return 0
-    for result in results:
-        print unicode(result).encode(preferred_encoding, 'replace')
-        covact = int(opts.covers)
-        if  covact == 1:
-            textcover = _('No cover found!')
-            if check_for_cover(result.isbn):
-                textcover = _('A cover was found for this book')
-            print textcover
-        elif covact == 2:
-            cover_data, ext = cover_from_isbn(result.isbn)
-            cpath = result.isbn
-            if len(opts.coverspath):
-                cpath = os.path.normpath(opts.coverspath + '/' + result.isbn)
-            oname = os.path.abspath(cpath+'.'+ext)
-            open(oname, 'wb').write(cover_data)
-            print _('Cover saved to file '), oname
-        print
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/ebooks/metadata/sources/base.py

@@ -307,7 +307,7 @@ class Source(Plugin):
             title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in
             [
                 # Remove things like: (2010) (Omnibus) etc.
-                (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''),
+                (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''),
                 # Remove any strings that contain the substring edition inside
                 # parentheses
                 (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''),

src/calibre/ebooks/metadata/sources/cli.py

@@ -19,13 +19,8 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ebooks.metadata.sources.base import create_log
 from calibre.ebooks.metadata.sources.identify import identify
 from calibre.ebooks.metadata.sources.covers import download_cover
-from calibre.utils.config import test_eight_code
 
 def option_parser():
-    if not test_eight_code:
-        from calibre.ebooks.metadata.fetch import option_parser
-        return option_parser()
-
     parser = OptionParser(textwrap.dedent(
         '''\
         %prog [options]
@@ -48,9 +43,6 @@ def option_parser():
     return parser
 
 def main(args=sys.argv):
-    if not test_eight_code:
-        from calibre.ebooks.metadata.fetch import main
-        return main(args)
     parser = option_parser()
     opts, args = parser.parse_args(args)
 

src/calibre/ebooks/metadata/sources/identify.py

@@ -400,6 +400,9 @@ def identify(log, abort, # {{{
                     and plugin.get_cached_cover_url(result.identifiers) is not
                     None)
             result.identify_plugin = plugin
+            if msprefs['txt_comments']:
+                if plugin.has_html_comments and result.comments:
+                    result.comments = html2text(r.comments)
 
     log('The identify phase took %.2f seconds'%(time.time() - start_time))
     log('The longest time (%f) was taken by:'%longest, lp)
@@ -410,10 +413,6 @@ def identify(log, abort, # {{{
     log('We have %d merged results, merging took: %.2f seconds' %
             (len(results), time.time() - start_time))
 
-    if msprefs['txt_comments']:
-        for r in results:
-            if r.identify_plugin.has_html_comments and r.comments:
-                r.comments = html2text(r.comments)
 
     max_tags = msprefs['max_tags']
     for r in results:

src/calibre/ebooks/metadata/sources/isbndb.py

@@ -81,7 +81,7 @@ class ISBNDB(Source):
             author_tokens = self.get_author_tokens(authors,
                     only_first_author=True)
             tokens += author_tokens
-            tokens = [quote(t) for t in tokens]
+            tokens = [quote(t.encode('utf-8') if isinstance(t, unicode) else t) for t in tokens]
             q = '+'.join(tokens)
             q = 'index1=combined&value1='+q
 

src/calibre/ebooks/oeb/base.py

@@ -16,7 +16,7 @@ from urllib import unquote as urlunquote
 from lxml import etree, html
 from calibre.constants import filesystem_encoding, __version__
 from calibre.translations.dynamic import translate
-from calibre.ebooks.chardet import xml_to_unicode
+from calibre.ebooks.chardet import xml_to_unicode, strip_encoding_declarations
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.ebooks.conversion.preprocess import CSSPreProcessor
 from calibre import isbytestring, as_unicode, get_types_map
@@ -853,6 +853,7 @@ class Manifest(object):
             self.oeb.log.debug('Parsing', self.href, '...')
             # Convert to Unicode and normalize line endings
             data = self.oeb.decode(data)
+            data = strip_encoding_declarations(data)
             data = self.oeb.html_preprocessor(data)
             # There could be null bytes in data if it had � entities in it
             data = data.replace('\0', '')

src/calibre/ebooks/oeb/stylizer.py

@@ -125,7 +125,19 @@ class Stylizer(object):
     def __init__(self, tree, path, oeb, opts, profile=None,
             extra_css='', user_css=''):
         self.oeb, self.opts = oeb, opts
-        self.profile = opts.input_profile
+        self.profile = profile
+        if self.profile is None:
+            # Use the default profile. This should really be using
+            # opts.output_profile, but I don't want to risk changing it, as
+            # doing so might well have hard to debug font size effects.
+            from calibre.customize.ui import output_profiles
+            for x in output_profiles():
+                if x.short_name == 'default':
+                    self.profile = x
+                    break
+        if self.profile is None:
+            # Just in case the default profile is removed in the future :)
+            self.profile = opts.output_profile
         self.logger = oeb.logger
         item = oeb.manifest.hrefs[path]
         basename = os.path.basename(path)

src/calibre/ebooks/pdb/plucker/reader.py

@@ -360,7 +360,7 @@ class Reader(FormatReader):
         # plugin assemble the order based on hyperlinks.
         with CurrentDir(output_dir):
             for uid, num in self.uid_text_secion_number.items():
-                self.log.debug(_('Writing record with uid: %s as %s.html' % (uid, uid)))
+                self.log.debug('Writing record with uid: %s as %s.html' % (uid, uid))
                 with open('%s.html' % uid, 'wb') as htmlf:
                     html = u'<html><body>'
                     section_header, section_data = self.sections[num]
@@ -466,7 +466,7 @@ class Reader(FormatReader):
             if not home_html:
                 home_html = self.uid_text_secion_number.items()[0][0]
         except:
-            raise Exception(_('Could not determine home.html'))
+            raise Exception('Could not determine home.html')
         # Generate oeb from html conversion.
         oeb = html_input.convert(open('%s.html' % home_html, 'rb'), self.options, 'html', self.log, {})
         self.options.debug_pipeline = odi

src/calibre/gui2/actions/add.py

@@ -20,9 +20,8 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.utils.filenames import ascii_filename
 from calibre.constants import preferred_encoding, filesystem_encoding
 from calibre.gui2.actions import InterfaceAction
-from calibre.gui2 import config, question_dialog
+from calibre.gui2 import question_dialog
 from calibre.ebooks.metadata import MetaInformation
-from calibre.utils.config import test_eight_code
 from calibre.ebooks.metadata.sources.base import msprefs
 
 def get_filters():
@@ -180,26 +179,17 @@ class AddAction(InterfaceAction):
             except IndexError:
                 self.gui.library_view.model().books_added(self.isbn_add_dialog.value)
                 self.isbn_add_dialog.accept()
-                if test_eight_code:
+                orig = msprefs['ignore_fields']
-                    orig = msprefs['ignore_fields']
+                new = list(orig)
-                    new = list(orig)
+                for x in ('title', 'authors'):
-                    for x in ('title', 'authors'):
+                    if x in new:
-                        if x in new:
+                        new.remove(x)
-                            new.remove(x)
+                msprefs['ignore_fields'] = new
-                    msprefs['ignore_fields'] = new
+                try:
-                    try:
+                    self.gui.iactions['Edit Metadata'].download_metadata(
-                        self.gui.iactions['Edit Metadata'].download_metadata(
+                        ids=self.add_by_isbn_ids)
-                            ids=self.add_by_isbn_ids)
+                finally:
-                    finally:
+                    msprefs['ignore_fields'] = orig
-                        msprefs['ignore_fields'] = orig
-                else:
-                    orig = config['overwrite_author_title_metadata']
-                    config['overwrite_author_title_metadata'] = True
-                    try:
-                        self.gui.iactions['Edit Metadata'].do_download_metadata(
-                                self.add_by_isbn_ids)
-                    finally:
-                        config['overwrite_author_title_metadata'] = orig
                 return
 
 

src/calibre/gui2/actions/edit_metadata.py

@@ -10,15 +10,13 @@ from functools import partial
 
 from PyQt4.Qt import Qt, QMenu, QModelIndex, QTimer
 
-from calibre.gui2 import error_dialog, config, Dispatcher, question_dialog
+from calibre.gui2 import error_dialog, Dispatcher, question_dialog
-from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.icu import sort_key
-from calibre.utils.config import test_eight_code
 
 class EditMetadataAction(InterfaceAction):
 
@@ -36,22 +34,8 @@ class EditMetadataAction(InterfaceAction):
         md.addAction(_('Edit metadata in bulk'),
                 partial(self.edit_metadata, False, bulk=True))
         md.addSeparator()
-        if test_eight_code:
+        md.addAction(_('Download metadata and covers'), self.download_metadata,
-            dall = self.download_metadata
-        else:
-            dall = partial(self.download_metadata_old, False, covers=True)
-            dident = partial(self.download_metadata_old, False, covers=False)
-            dcovers = partial(self.download_metadata_old, False, covers=True,
-                    set_metadata=False, set_social_metadata=False)
-
-        md.addAction(_('Download metadata and covers'), dall,
                 Qt.ControlModifier+Qt.Key_D)
-        if not test_eight_code:
-            md.addAction(_('Download only metadata'), dident)
-            md.addAction(_('Download only covers'), dcovers)
-            md.addAction(_('Download only social metadata'),
-                partial(self.download_metadata_old, False, covers=False,
-                    set_metadata=False, set_social_metadata=True))
         self.metadata_menu = md
 
         mb = QMenu()
@@ -88,7 +72,7 @@ class EditMetadataAction(InterfaceAction):
                             _('No books selected'), show=True)
             db = self.gui.library_view.model().db
             ids = [db.id(row.row()) for row in rows]
-        from calibre.gui2.metadata.bulk_download2 import start_download
+        from calibre.gui2.metadata.bulk_download import start_download
         start_download(self.gui, ids,
                 Dispatcher(self.metadata_downloaded))
 
@@ -96,7 +80,7 @@ class EditMetadataAction(InterfaceAction):
         if job.failed:
             self.gui.job_exception(job, dialog_title=_('Failed to download metadata'))
             return
-        from calibre.gui2.metadata.bulk_download2 import get_job_details
+        from calibre.gui2.metadata.bulk_download import get_job_details
         id_map, failed_ids, failed_covers, all_failed, det_msg = \
                                             get_job_details(job)
         if all_failed:
@@ -112,8 +96,9 @@ class EditMetadataAction(InterfaceAction):
         show_copy_button = False
         if failed_ids or failed_covers:
             show_copy_button = True
+            num = len(failed_ids.union(failed_covers))
             msg += '<p>'+_('Could not download metadata and/or covers for %d of the books. Click'
-                    ' "Show details" to see which books.')%len(failed_ids)
+                    ' "Show details" to see which books.')%num
 
         payload = (id_map, failed_ids, failed_covers)
         from calibre.gui2.dialogs.message_box import ProceedNotification
@@ -158,49 +143,6 @@ class EditMetadataAction(InterfaceAction):
 
         self.apply_metadata_changes(id_map)
 
-    def download_metadata_old(self, checked, covers=True, set_metadata=True,
-            set_social_metadata=None):
-        rows = self.gui.library_view.selectionModel().selectedRows()
-        if not rows or len(rows) == 0:
-            d = error_dialog(self.gui, _('Cannot download metadata'),
-                             _('No books selected'))
-            d.exec_()
-            return
-        db = self.gui.library_view.model().db
-        ids = [db.id(row.row()) for row in rows]
-        self.do_download_metadata(ids, covers=covers,
-                set_metadata=set_metadata,
-                set_social_metadata=set_social_metadata)
-
-    def do_download_metadata(self, ids, covers=True, set_metadata=True,
-            set_social_metadata=None):
-        m = self.gui.library_view.model()
-        db = m.db
-        if set_social_metadata is None:
-            get_social_metadata = config['get_social_metadata']
-        else:
-            get_social_metadata = set_social_metadata
-        from calibre.gui2.metadata.bulk_download import DoDownload
-        if set_social_metadata is not None and set_social_metadata:
-            x = _('social metadata')
-        else:
-            x = _('covers') if covers and not set_metadata else _('metadata')
-        title = _('Downloading {0} for {1} book(s)').format(x, len(ids))
-        self._download_book_metadata = DoDownload(self.gui, title, db, ids,
-                get_covers=covers, set_metadata=set_metadata,
-                get_social_metadata=get_social_metadata)
-        m.stop_metadata_backup()
-        try:
-            self._download_book_metadata.exec_()
-        finally:
-            m.start_metadata_backup()
-        cr = self.gui.library_view.currentIndex().row()
-        x = self._download_book_metadata
-        if x.updated:
-            self.gui.library_view.model().refresh_ids(
-                x.updated, cr)
-            if self.gui.cover_flow:
-                self.gui.cover_flow.dataChanged()
     # }}}
 
     def edit_metadata(self, checked, bulk=None):
@@ -227,9 +169,7 @@ class EditMetadataAction(InterfaceAction):
                 list(range(self.gui.library_view.model().rowCount(QModelIndex())))
             current_row = row_list.index(cr)
 
-        func = (self.do_edit_metadata if test_eight_code else
+        changed, rows_to_refresh = self.do_edit_metadata(row_list, current_row)
-                    self.do_edit_metadata_old)
-        changed, rows_to_refresh = func(row_list, current_row)
 
         m = self.gui.library_view.model()
 
@@ -244,36 +184,6 @@ class EditMetadataAction(InterfaceAction):
             m.current_changed(current, previous)
             self.gui.tags_view.recount()
 
-    def do_edit_metadata_old(self, row_list, current_row):
-        changed = set([])
-        db = self.gui.library_view.model().db
-
-        while True:
-            prev = next_ = None
-            if current_row > 0:
-                prev = db.title(row_list[current_row-1])
-            if current_row < len(row_list) - 1:
-                next_ = db.title(row_list[current_row+1])
-
-            d = MetadataSingleDialog(self.gui, row_list[current_row], db,
-                    prev=prev, next_=next_)
-            d.view_format.connect(lambda
-                    fmt:self.gui.iactions['View'].view_format(row_list[current_row],
-                        fmt))
-            ret = d.exec_()
-            d.break_cycles()
-            if ret != d.Accepted:
-                break
-
-            changed.add(d.id)
-            self.gui.library_view.model().refresh_ids(list(d.books_to_refresh))
-            if d.row_delta == 0:
-                break
-            current_row += d.row_delta
-            self.gui.library_view.set_current_row(current_row)
-            self.gui.library_view.scroll_to_row(current_row)
-        return changed, set()
-
     def do_edit_metadata(self, row_list, current_row):
         from calibre.gui2.metadata.single import edit_metadata
         db = self.gui.library_view.model().db
@@ -613,6 +523,7 @@ class EditMetadataAction(InterfaceAction):
                 self.applied_ids, cr)
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()
+            self.gui.tags_view.recount()
 
         self.apply_id_map = []
         self.apply_pd = None

src/calibre/gui2/dialogs/fetch_metadata.py

@@ -1,271 +0,0 @@
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-'''
-GUI for fetching metadata from servers.
-'''
-
-import time
-from threading import Thread
-
-from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \
-                         QAbstractTableModel, QCoreApplication, QTimer
-from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon
-
-from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
-from calibre.gui2 import error_dialog, NONE, info_dialog, config
-from calibre.gui2.widgets import ProgressIndicator
-from calibre import strftime, force_unicode
-from calibre.customize.ui import get_isbndb_key, set_isbndb_key
-from calibre.utils.icu import sort_key
-
-_hung_fetchers = set([])
-
-class Fetcher(Thread):
-
-    def __init__(self, title, author, publisher, isbn, key):
-        Thread.__init__(self)
-        self.daemon = True
-        self.title = title
-        self.author = author
-        self.publisher = publisher
-        self.isbn = isbn
-        self.key = key
-        self.results, self.exceptions = [], []
-
-    def run(self):
-        from calibre.ebooks.metadata.fetch import search
-        self.results, self.exceptions = search(self.title, self.author,
-                                               self.publisher, self.isbn,
-                                               self.key if self.key else None)
-
-
-class Matches(QAbstractTableModel):
-
-    def __init__(self, matches):
-        self.matches = matches
-        self.yes_icon = QVariant(QIcon(I('ok.png')))
-        QAbstractTableModel.__init__(self)
-
-    def rowCount(self, *args):
-        return len(self.matches)
-
-    def columnCount(self, *args):
-        return 8
-
-    def headerData(self, section, orientation, role):
-        if role != Qt.DisplayRole:
-            return NONE
-        text = ""
-        if orientation == Qt.Horizontal:
-            if   section == 0: text = _("Title")
-            elif section == 1: text = _("Author(s)")
-            elif section == 2: text = _("Author Sort")
-            elif section == 3: text = _("Publisher")
-            elif section == 4: text = _("ISBN")
-            elif section == 5: text = _("Published")
-            elif section == 6: text = _("Has Cover")
-            elif section == 7: text = _("Has Summary")
-
-            return QVariant(text)
-        else:
-            return QVariant(section+1)
-
-    def summary(self, row):
-        return self.matches[row].comments
-
-    def data_as_text(self, book, col):
-        if col == 0 and book.title is not None:
-            return book.title
-        elif col == 1:
-            return ', '.join(book.authors)
-        elif col == 2 and book.author_sort is not None:
-            return book.author_sort
-        elif col == 3 and book.publisher is not None:
-            return book.publisher
-        elif col == 4 and book.isbn is not None:
-            return book.isbn
-        elif col == 5 and hasattr(book.pubdate, 'timetuple'):
-            return strftime('%b %Y', book.pubdate.timetuple())
-        elif col == 6 and book.has_cover:
-            return 'y'
-        elif col == 7 and book.comments:
-            return 'y'
-        return ''
-
-    def data(self, index, role):
-        row, col = index.row(), index.column()
-        book = self.matches[row]
-        if role == Qt.DisplayRole:
-            res = self.data_as_text(book, col)
-            if col <= 5 and res:
-                return QVariant(res)
-            return NONE
-        elif role == Qt.DecorationRole:
-            if col == 6 and book.has_cover:
-                return self.yes_icon
-            if col == 7 and book.comments:
-                return self.yes_icon
-        return NONE
-
-    def sort(self, col, order, reset=True):
-        if not self.matches:
-            return
-        descending = order == Qt.DescendingOrder
-        self.matches.sort(None,
-            lambda x: sort_key(unicode(force_unicode(self.data_as_text(x, col)))),
-            descending)
-        if reset:
-            self.reset()
-
-class FetchMetadata(QDialog, Ui_FetchMetadata):
-
-    HANG_TIME = 75 #seconds
-
-    queue_reject = pyqtSignal()
-
-    def __init__(self, parent, isbn, title, author, publisher, timeout):
-        QDialog.__init__(self, parent)
-        Ui_FetchMetadata.__init__(self)
-        self.setupUi(self)
-
-        for fetcher in list(_hung_fetchers):
-            if not fetcher.is_alive():
-                _hung_fetchers.remove(fetcher)
-
-        self.pi = ProgressIndicator(self)
-        self.timeout = timeout
-        QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata)
-        self.queue_reject.connect(self.reject, Qt.QueuedConnection)
-
-        isbndb_key = get_isbndb_key()
-        if not isbndb_key:
-            isbndb_key = ''
-        self.key.setText(isbndb_key)
-
-        self.setWindowTitle(title if title else _('Unknown'))
-        self.isbn = isbn
-        self.title = title
-        self.author = author.strip()
-        self.publisher = publisher
-        self.previous_row = None
-        self.warning.setVisible(False)
-        self.connect(self.matches, SIGNAL('activated(QModelIndex)'), self.chosen)
-        self.connect(self.matches, SIGNAL('entered(QModelIndex)'),
-                     self.show_summary)
-        self.matches.setMouseTracking(True)
-        # Enabling sorting and setting a sort column will not change the initial
-        # order of the results, as they are filled in later
-        self.matches.setSortingEnabled(True)
-        self.matches.horizontalHeader().sectionClicked.connect(self.show_sort_indicator)
-        self.matches.horizontalHeader().setSortIndicatorShown(False)
-        self.fetch_metadata()
-        self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
-        self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata'])
-        self.opt_auto_download_cover.setChecked(config['auto_download_cover'])
-
-    def show_summary(self, current, *args):
-        row  = current.row()
-        if row != self.previous_row:
-            summ =  self.model.summary(row)
-            self.summary.setText(summ if summ else '')
-            self.previous_row = row
-
-    def fetch_metadata(self):
-        self.warning.setVisible(False)
-        key = str(self.key.text())
-        if key:
-            set_isbndb_key(key)
-        else:
-            key = None
-        title = author = publisher = isbn = None
-        if self.isbn:
-            isbn = self.isbn
-        if self.title:
-            title = self.title
-        if self.author and not self.author == _('Unknown'):
-            author = self.author
-        self.fetch.setEnabled(False)
-        self.setCursor(Qt.WaitCursor)
-        QCoreApplication.instance().processEvents()
-        self.fetcher = Fetcher(title, author, publisher, isbn, key)
-        self.fetcher.start()
-        self.pi.start(_('Finding metadata...'))
-        self._hangcheck = QTimer(self)
-        self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck,
-                Qt.QueuedConnection)
-        self.start_time = time.time()
-        self._hangcheck.start(100)
-
-    def hangcheck(self):
-        if self.fetcher.is_alive() and \
-                time.time() - self.start_time < self.HANG_TIME:
-            return
-        self._hangcheck.stop()
-        try:
-            if self.fetcher.is_alive():
-                error_dialog(self, _('Could not find metadata'),
-                             _('The metadata download seems to have stalled. '
-                               'Try again later.')).exec_()
-                self.terminate()
-                return self.queue_reject.emit()
-            self.model = Matches(self.fetcher.results)
-            warnings = [(x[0], force_unicode(x[1])) for x in \
-                            self.fetcher.exceptions if x[1] is not None]
-            if warnings:
-                warnings='<br>'.join(['<b>%s</b>: %s'%(name, exc) for name,exc in warnings])
-                self.warning.setText('<p><b>'+ _('Warning')+':</b>'+\
-                               _('Could not fetch metadata from:')+\
-                               '<br>'+warnings+'</p>')
-                self.warning.setVisible(True)
-            if self.model.rowCount() < 1:
-                info_dialog(self, _('No metadata found'),
-                     _('No metadata found, try adjusting the title and author '
-                       'and/or removing the ISBN.')).exec_()
-                self.reject()
-                return
-
-            self.matches.setModel(self.model)
-            QObject.connect(self.matches.selectionModel(),
-                        SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
-                        self.show_summary)
-            self.model.reset()
-            self.matches.selectionModel().select(self.model.index(0, 0),
-                                  QItemSelectionModel.Select | QItemSelectionModel.Rows)
-            self.matches.setCurrentIndex(self.model.index(0, 0))
-        finally:
-            self.fetch.setEnabled(True)
-            self.unsetCursor()
-            self.matches.resizeColumnsToContents()
-            self.pi.stop()
-
-    def terminate(self):
-        if hasattr(self, 'fetcher') and self.fetcher.is_alive():
-            _hung_fetchers.add(self.fetcher)
-        if hasattr(self, '_hangcheck') and self._hangcheck.isActive():
-            self._hangcheck.stop()
-        # Save value of auto_download_cover, since this is the only place it can
-        # be set. The values of the other options can be set in
-        # Preferences->Behavior and should not be set here as they affect bulk
-        # downloading as well.
-        if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']:
-           config.set('auto_download_cover', self.opt_auto_download_cover.isChecked())
-
-    def __enter__(self, *args):
-        return self
-
-    def __exit__(self, *args):
-        self.terminate()
-
-    def selected_book(self):
-        try:
-            return self.matches.model().matches[self.matches.currentIndex().row()]
-        except:
-            return None
-
-    def chosen(self, index):
-        self.matches.setCurrentIndex(index)
-        self.accept()
-
-    def show_sort_indicator(self, *args):
-        self.matches.horizontalHeader().setSortIndicatorShown(True)
-

src/calibre/gui2/dialogs/fetch_metadata.ui

@@ -1,179 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>FetchMetadata</class>
- <widget class="QDialog" name="FetchMetadata">
-  <property name="windowModality">
-   <enum>Qt::WindowModal</enum>
-  </property>
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>890</width>
-    <height>642</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>Fetch metadata</string>
-  </property>
-  <property name="windowIcon">
-   <iconset resource="../../../../resources/images.qrc">
-    <normaloff>:/images/metadata.png</normaloff>:/images/metadata.png</iconset>
-  </property>
-  <layout class="QVBoxLayout">
-   <item>
-    <widget class="QLabel" name="tlabel">
-     <property name="text">
-      <string>&lt;p&gt;calibre can find metadata for your books from two locations: &lt;b&gt;Google Books&lt;/b&gt; and &lt;b&gt;isbndb.com&lt;/b&gt;. &lt;p&gt;To use isbndb.com you must sign up for a &lt;a href=&quot;http://www.isbndb.com&quot;&gt;free account&lt;/a&gt; and enter your access key below.</string>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignCenter</set>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-     <property name="openExternalLinks">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <layout class="QHBoxLayout">
-     <item>
-      <widget class="QLabel" name="label_2">
-       <property name="text">
-        <string>&amp;Access Key:</string>
-       </property>
-       <property name="buddy">
-        <cstring>key</cstring>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QLineEdit" name="key"/>
-     </item>
-     <item>
-      <widget class="QPushButton" name="fetch">
-       <property name="text">
-        <string>Fetch</string>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-   <item>
-    <widget class="QLabel" name="warning">
-     <property name="text">
-      <string/>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QGroupBox" name="groupBox">
-     <property name="title">
-      <string>Matches</string>
-     </property>
-     <layout class="QVBoxLayout">
-      <item>
-       <widget class="QLabel" name="label_3">
-        <property name="text">
-         <string>Select the book that most closely matches your copy from the list below</string>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QTableView" name="matches">
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-          <horstretch>0</horstretch>
-          <verstretch>1</verstretch>
-         </sizepolicy>
-        </property>
-        <property name="alternatingRowColors">
-         <bool>true</bool>
-        </property>
-        <property name="selectionMode">
-         <enum>QAbstractItemView::SingleSelection</enum>
-        </property>
-        <property name="selectionBehavior">
-         <enum>QAbstractItemView::SelectRows</enum>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QTextBrowser" name="summary"/>
-      </item>
-     </layout>
-    </widget>
-   </item>
-   <item>
-    <widget class="QCheckBox" name="opt_overwrite_author_title_metadata">
-     <property name="text">
-      <string>Overwrite author and title with author and title of selected book</string>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QCheckBox" name="opt_get_social_metadata">
-     <property name="text">
-      <string>Download &amp;social metadata (tags/rating/etc.) for the selected book</string>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QCheckBox" name="opt_auto_download_cover">
-     <property name="text">
-      <string>Automatically download the cover, if available</string>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QDialogButtonBox" name="buttonBox">
-     <property name="standardButtons">
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     </property>
-    </widget>
-   </item>
-  </layout>
- </widget>
- <resources>
-  <include location="../../../../resources/images.qrc"/>
- </resources>
- <connections>
-  <connection>
-   <sender>buttonBox</sender>
-   <signal>accepted()</signal>
-   <receiver>FetchMetadata</receiver>
-   <slot>accept()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>460</x>
-     <y>599</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>657</x>
-     <y>530</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>buttonBox</sender>
-   <signal>rejected()</signal>
-   <receiver>FetchMetadata</receiver>
-   <slot>reject()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>417</x>
-     <y>599</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>0</x>
-     <y>491</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
-</ui>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -1,937 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>MetadataSingleDialog</class>
- <widget class="QDialog" name="MetadataSingleDialog">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>994</width>
-    <height>716</height>
-   </rect>
-  </property>
-  <property name="sizePolicy">
-   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
-    <horstretch>0</horstretch>
-    <verstretch>0</verstretch>
-   </sizepolicy>
-  </property>
-  <property name="windowTitle">
-   <string>Edit Meta Information</string>
-  </property>
-  <property name="windowIcon">
-   <iconset resource="../../../../resources/images.qrc">
-    <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
-  </property>
-  <property name="sizeGripEnabled">
-   <bool>true</bool>
-  </property>
-  <property name="modal">
-   <bool>true</bool>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout_6">
-   <item>
-    <widget class="QScrollArea" name="scrollArea">
-     <property name="frameShape">
-      <enum>QFrame::NoFrame</enum>
-     </property>
-     <property name="widgetResizable">
-      <bool>true</bool>
-     </property>
-     <widget class="QWidget" name="scrollAreaWidgetContents">
-      <property name="geometry">
-       <rect>
-        <x>0</x>
-        <y>0</y>
-        <width>986</width>
-        <height>677</height>
-       </rect>
-      </property>
-      <layout class="QVBoxLayout" name="verticalLayout_5">
-       <property name="margin">
-        <number>0</number>
-       </property>
-       <item>
-        <widget class="QTabWidget" name="central_widget">
-         <property name="minimumSize">
-          <size>
-           <width>800</width>
-           <height>665</height>
-          </size>
-         </property>
-         <property name="currentIndex">
-          <number>0</number>
-         </property>
-         <widget class="QWidget" name="central_tabWidgetPage1">
-          <attribute name="title">
-           <string>&amp;Basic metadata</string>
-          </attribute>
-          <layout class="QGridLayout" name="gridLayout_5">
-           <item row="0" column="0">
-            <widget class="QSplitter" name="splitter">
-             <property name="orientation">
-              <enum>Qt::Horizontal</enum>
-             </property>
-             <widget class="QWidget" name="layoutWidget">
-              <layout class="QVBoxLayout">
-               <item>
-                <widget class="QGroupBox" name="meta_box">
-                 <property name="title">
-                  <string>Meta information</string>
-                 </property>
-                 <layout class="QGridLayout" name="gridLayout_3">
-                  <item row="0" column="0">
-                   <widget class="QLabel" name="label">
-                    <property name="text">
-                     <string>&amp;Title: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>title</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="0" column="1">
-                   <widget class="EnLineEdit" name="title">
-                    <property name="toolTip">
-                     <string>Change the title of this book</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="0" column="2" rowspan="4">
-                   <layout class="QVBoxLayout" name="verticalLayout_7">
-                    <item>
-                     <spacer name="verticalSpacer_3">
-                      <property name="orientation">
-                       <enum>Qt::Vertical</enum>
-                      </property>
-                      <property name="sizeHint" stdset="0">
-                       <size>
-                        <width>20</width>
-                        <height>40</height>
-                       </size>
-                      </property>
-                     </spacer>
-                    </item>
-                    <item>
-                     <widget class="QToolButton" name="auto_title_sort">
-                      <property name="toolTip">
-                       <string>Automatically create the title sort entry based on the current title entry.
-Using this button to create title sort will change title sort from red to green.</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset>
-                      </property>
-                     </widget>
-                    </item>
-                    <item>
-                     <spacer name="verticalSpacer">
-                      <property name="orientation">
-                       <enum>Qt::Vertical</enum>
-                      </property>
-                      <property name="sizeHint" stdset="0">
-                       <size>
-                        <width>20</width>
-                        <height>40</height>
-                       </size>
-                      </property>
-                     </spacer>
-                    </item>
-                    <item>
-                     <widget class="QToolButton" name="swap_button">
-                      <property name="toolTip">
-                       <string>Swap the author and title</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/swap.png</normaloff>:/images/swap.png</iconset>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>16</width>
-                        <height>16</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                    <item>
-                     <spacer name="verticalSpacer_2">
-                      <property name="orientation">
-                       <enum>Qt::Vertical</enum>
-                      </property>
-                      <property name="sizeHint" stdset="0">
-                       <size>
-                        <width>20</width>
-                        <height>40</height>
-                       </size>
-                      </property>
-                     </spacer>
-                    </item>
-                    <item>
-                     <widget class="QToolButton" name="auto_author_sort">
-                      <property name="toolTip">
-                       <string>Automatically create the author sort entry based on the current author entry.
-Using this button to create author sort will change author sort from red to green.</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset>
-                      </property>
-                     </widget>
-                    </item>
-                    <item>
-                     <spacer name="verticalSpacer_4">
-                      <property name="orientation">
-                       <enum>Qt::Vertical</enum>
-                      </property>
-                      <property name="sizeHint" stdset="0">
-                       <size>
-                        <width>20</width>
-                        <height>40</height>
-                       </size>
-                      </property>
-                     </spacer>
-                    </item>
-                   </layout>
-                  </item>
-                  <item row="1" column="0">
-                   <widget class="QLabel" name="label">
-                    <property name="text">
-                     <string>Title &amp;sort: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>title_sort</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="1" column="1">
-                   <widget class="EnLineEdit" name="title_sort">
-                    <property name="toolTip">
-                     <string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="2" column="0">
-                   <widget class="QLabel" name="label_2">
-                    <property name="text">
-                     <string>&amp;Author(s): </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>authors</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="2" column="1">
-                   <widget class="MultiCompleteComboBox" name="authors">
-                    <property name="editable">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="3" column="0">
-                   <widget class="QLabel" name="label_8">
-                    <property name="text">
-                     <string>Author S&amp;ort: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>author_sort</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="3" column="1">
-                   <widget class="EnLineEdit" name="author_sort">
-                    <property name="toolTip">
-                     <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
-If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="4" column="0">
-                   <widget class="QLabel" name="label_6">
-                    <property name="text">
-                     <string>&amp;Rating:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>rating</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="4" column="1" colspan="2">
-                   <widget class="QSpinBox" name="rating">
-                    <property name="toolTip">
-                     <string>Rating of this book. 0-5 stars</string>
-                    </property>
-                    <property name="whatsThis">
-                     <string>Rating of this book. 0-5 stars</string>
-                    </property>
-                    <property name="buttonSymbols">
-                     <enum>QAbstractSpinBox::PlusMinus</enum>
-                    </property>
-                    <property name="suffix">
-                     <string> stars</string>
-                    </property>
-                    <property name="maximum">
-                     <number>5</number>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="5" column="0">
-                   <widget class="QLabel" name="label_3">
-                    <property name="text">
-                     <string>&amp;Publisher: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>publisher</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="5" column="1" colspan="2">
-                   <widget class="MultiCompleteComboBox" name="publisher">
-                    <property name="editable">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="6" column="0">
-                   <widget class="QLabel" name="label_4">
-                    <property name="text">
-                     <string>Ta&amp;gs: </string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>tags</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="6" column="1">
-                   <layout class="QHBoxLayout" name="_2">
-                    <item>
-                     <widget class="MultiCompleteLineEdit" name="tags">
-                      <property name="toolTip">
-                       <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
-                      </property>
-                     </widget>
-                    </item>
-                   </layout>
-                  </item>
-                  <item row="6" column="2">
-                   <widget class="QToolButton" name="tag_editor_button">
-                    <property name="toolTip">
-                     <string>Open Tag Editor</string>
-                    </property>
-                    <property name="text">
-                     <string>Open Tag Editor</string>
-                    </property>
-                    <property name="icon">
-                     <iconset resource="../../../../resources/images.qrc">
-                      <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="7" column="0">
-                   <widget class="QLabel" name="label_7">
-                    <property name="text">
-                     <string>&amp;Series:</string>
-                    </property>
-                    <property name="textFormat">
-                     <enum>Qt::PlainText</enum>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>series</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="7" column="1">
-                   <layout class="QHBoxLayout" name="_3">
-                    <property name="spacing">
-                     <number>5</number>
-                    </property>
-                    <item>
-                     <widget class="MultiCompleteComboBox" name="series">
-                      <property name="toolTip">
-                       <string>List of known series. You can add new series.</string>
-                      </property>
-                      <property name="whatsThis">
-                       <string>List of known series. You can add new series.</string>
-                      </property>
-                      <property name="editable">
-                       <bool>true</bool>
-                      </property>
-                      <property name="insertPolicy">
-                       <enum>QComboBox::InsertAlphabetically</enum>
-                      </property>
-                     </widget>
-                    </item>
-                   </layout>
-                  </item>
-                  <item row="7" column="2">
-                   <widget class="QToolButton" name="remove_series_button">
-                    <property name="toolTip">
-                     <string>Remove unused series (Series that have no books)</string>
-                    </property>
-                    <property name="text">
-                     <string>...</string>
-                    </property>
-                    <property name="icon">
-                     <iconset resource="../../../../resources/images.qrc">
-                      <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="8" column="1" colspan="2">
-                   <widget class="QDoubleSpinBox" name="series_index">
-                    <property name="enabled">
-                     <bool>false</bool>
-                    </property>
-                    <property name="prefix">
-                     <string>Book </string>
-                    </property>
-                    <property name="maximum">
-                     <double>99999999.989999994635582</double>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="9" column="0">
-                   <widget class="QLabel" name="label_9">
-                    <property name="text">
-                     <string>IS&amp;BN:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>isbn</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="9" column="1" colspan="2">
-                   <widget class="QLineEdit" name="isbn"/>
-                  </item>
-                  <item row="10" column="0">
-                   <widget class="QLabel" name="label_11">
-                    <property name="text">
-                     <string>&amp;Date:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>date</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="10" column="1" colspan="2">
-                   <widget class="QDateEdit" name="date">
-                    <property name="displayFormat">
-                     <string>dd MMM yyyy</string>
-                    </property>
-                    <property name="calendarPopup">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="11" column="0">
-                   <widget class="QLabel" name="label_10">
-                    <property name="text">
-                     <string>Publishe&amp;d:</string>
-                    </property>
-                    <property name="alignment">
-                     <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-                    </property>
-                    <property name="buddy">
-                     <cstring>pubdate</cstring>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="11" column="1">
-                   <widget class="QDateEdit" name="pubdate">
-                    <property name="displayFormat">
-                     <string>MMM yyyy</string>
-                    </property>
-                    <property name="calendarPopup">
-                     <bool>true</bool>
-                    </property>
-                   </widget>
-                  </item>
-                  <item row="11" column="2">
-                   <widget class="QToolButton" name="clear_pubdate_button">
-                    <property name="toolTip">
-                     <string>Clear published date</string>
-                    </property>
-                    <property name="icon">
-                     <iconset resource="../../../../resources/images.qrc">
-                      <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
-                    </property>
-                   </widget>
-                  </item>
-                 </layout>
-                </widget>
-               </item>
-               <item>
-                <widget class="QPushButton" name="fetch_metadata_button">
-                 <property name="text">
-                  <string>&amp;Fetch metadata from server</string>
-                 </property>
-                </widget>
-               </item>
-               <item>
-                <spacer name="verticalSpacer_5">
-                 <property name="orientation">
-                  <enum>Qt::Vertical</enum>
-                 </property>
-                 <property name="sizeType">
-                  <enum>QSizePolicy::Fixed</enum>
-                 </property>
-                 <property name="sizeHint" stdset="0">
-                  <size>
-                   <width>20</width>
-                   <height>40</height>
-                  </size>
-                 </property>
-                </spacer>
-               </item>
-              </layout>
-             </widget>
-             <widget class="QWidget" name="layoutWidget_2">
-              <layout class="QVBoxLayout" name="verticalLayout_2">
-               <item>
-                <widget class="QGroupBox" name="bc_box">
-                 <property name="sizePolicy">
-                  <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
-                   <horstretch>0</horstretch>
-                   <verstretch>10</verstretch>
-                  </sizepolicy>
-                 </property>
-                 <property name="title">
-                  <string>Book Cover</string>
-                 </property>
-                 <layout class="QVBoxLayout" name="verticalLayout_4">
-                  <item>
-                   <widget class="ImageView" name="cover" native="true">
-                    <property name="sizePolicy">
-                     <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-                      <horstretch>0</horstretch>
-                      <verstretch>100</verstretch>
-                     </sizepolicy>
-                    </property>
-                   </widget>
-                  </item>
-                  <item>
-                   <layout class="QVBoxLayout" name="_4">
-                    <property name="spacing">
-                     <number>6</number>
-                    </property>
-                    <property name="sizeConstraint">
-                     <enum>QLayout::SetMaximumSize</enum>
-                    </property>
-                    <property name="margin">
-                     <number>0</number>
-                    </property>
-                    <item>
-                     <widget class="QLabel" name="label_5">
-                      <property name="text">
-                       <string>Change &amp;cover image:</string>
-                      </property>
-                      <property name="buddy">
-                       <cstring>cover_button</cstring>
-                      </property>
-                     </widget>
-                    </item>
-                    <item>
-                     <layout class="QHBoxLayout" name="_5">
-                      <property name="spacing">
-                       <number>6</number>
-                      </property>
-                      <property name="margin">
-                       <number>0</number>
-                      </property>
-                      <item>
-                       <widget class="QPushButton" name="cover_button">
-                        <property name="text">
-                         <string>&amp;Browse</string>
-                        </property>
-                        <property name="icon">
-                         <iconset resource="../../../../resources/images.qrc">
-                          <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
-                        </property>
-                       </widget>
-                      </item>
-                      <item>
-                       <widget class="QPushButton" name="trim_cover_button">
-                        <property name="toolTip">
-                         <string>Remove border (if any) from cover</string>
-                        </property>
-                        <property name="text">
-                         <string>T&amp;rim</string>
-                        </property>
-                        <property name="icon">
-                         <iconset resource="../../../../resources/images.qrc">
-                          <normaloff>:/images/trim.png</normaloff>:/images/trim.png</iconset>
-                        </property>
-                       </widget>
-                      </item>
-                      <item>
-                       <widget class="QPushButton" name="reset_cover">
-                        <property name="toolTip">
-                         <string>Reset cover to default</string>
-                        </property>
-                        <property name="text">
-                         <string>&amp;Remove</string>
-                        </property>
-                        <property name="icon">
-                         <iconset resource="../../../../resources/images.qrc">
-                          <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
-                        </property>
-                       </widget>
-                      </item>
-                     </layout>
-                    </item>
-                   </layout>
-                  </item>
-                  <item>
-                   <layout class="QHBoxLayout" name="_6">
-                    <item>
-                     <widget class="QPushButton" name="fetch_cover_button">
-                      <property name="text">
-                       <string>Download co&amp;ver</string>
-                      </property>
-                     </widget>
-                    </item>
-                    <item>
-                     <widget class="QPushButton" name="generate_cover_button">
-                      <property name="toolTip">
-                       <string>Generate a default cover based on the title and author</string>
-                      </property>
-                      <property name="text">
-                       <string>&amp;Generate cover</string>
-                      </property>
-                     </widget>
-                    </item>
-                   </layout>
-                  </item>
-                 </layout>
-                </widget>
-               </item>
-              </layout>
-             </widget>
-             <widget class="QWidget" name="layoutWidget">
-              <layout class="QVBoxLayout" name="verticalLayout_3">
-               <item>
-                <widget class="QGroupBox" name="af_group_box">
-                 <property name="sizePolicy">
-                  <sizepolicy hsizetype="Preferred" vsizetype="Minimum">
-                   <horstretch>0</horstretch>
-                   <verstretch>0</verstretch>
-                  </sizepolicy>
-                 </property>
-                 <property name="title">
-                  <string>Available Formats</string>
-                 </property>
-                 <layout class="QVBoxLayout" name="verticalLayout">
-                  <item>
-                   <layout class="QGridLayout" name="gridLayout">
-                    <item row="0" column="1" rowspan="3">
-                     <widget class="FormatList" name="formats">
-                      <property name="sizePolicy">
-                       <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
-                        <horstretch>0</horstretch>
-                        <verstretch>0</verstretch>
-                       </sizepolicy>
-                      </property>
-                      <property name="maximumSize">
-                       <size>
-                        <width>16777215</width>
-                        <height>140</height>
-                       </size>
-                      </property>
-                      <property name="baseSize">
-                       <size>
-                        <width>100</width>
-                        <height>0</height>
-                       </size>
-                      </property>
-                      <property name="dragDropMode">
-                       <enum>QAbstractItemView::DropOnly</enum>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>64</width>
-                        <height>64</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                    <item row="0" column="2">
-                     <widget class="QToolButton" name="add_format_button">
-                      <property name="toolTip">
-                       <string>Add a new format for this book to the database</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/add_book.png</normaloff>:/images/add_book.png</iconset>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>32</width>
-                        <height>32</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                    <item row="2" column="2">
-                     <widget class="QToolButton" name="remove_format_button">
-                      <property name="toolTip">
-                       <string>Remove the selected formats for this book from the database.</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>32</width>
-                        <height>32</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                    <item row="0" column="0">
-                     <widget class="QToolButton" name="button_set_cover">
-                      <property name="toolTip">
-                       <string>Set the cover for the book from the selected format</string>
-                      </property>
-                      <property name="text">
-                       <string>...</string>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/book.png</normaloff>:/images/book.png</iconset>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>32</width>
-                        <height>32</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                    <item row="2" column="0">
-                     <widget class="QToolButton" name="button_set_metadata">
-                      <property name="toolTip">
-                       <string>Update metadata from the metadata in the selected format</string>
-                      </property>
-                      <property name="text">
-                       <string/>
-                      </property>
-                      <property name="icon">
-                       <iconset resource="../../../../resources/images.qrc">
-                        <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
-                      </property>
-                      <property name="iconSize">
-                       <size>
-                        <width>32</width>
-                        <height>32</height>
-                       </size>
-                      </property>
-                     </widget>
-                    </item>
-                   </layout>
-                  </item>
-                 </layout>
-                 <zorder></zorder>
-                </widget>
-               </item>
-               <item>
-                <widget class="QGroupBox" name="groupBox">
-                 <property name="sizePolicy">
-                  <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
-                   <horstretch>0</horstretch>
-                   <verstretch>10</verstretch>
-                  </sizepolicy>
-                 </property>
-                 <property name="title">
-                  <string>&amp;Comments</string>
-                 </property>
-                 <layout class="QVBoxLayout" name="verticalLayout_8">
-                  <property name="margin">
-                   <number>0</number>
-                  </property>
-                  <item>
-                   <widget class="Editor" name="comments" native="true"/>
-                  </item>
-                 </layout>
-                </widget>
-               </item>
-              </layout>
-             </widget>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-         <widget class="QWidget" name="tab">
-          <attribute name="title">
-           <string>&amp;Custom metadata</string>
-          </attribute>
-          <layout class="QGridLayout" name="gridLayout_2"/>
-         </widget>
-        </widget>
-       </item>
-      </layout>
-     </widget>
-    </widget>
-   </item>
-   <item>
-    <widget class="QDialogButtonBox" name="button_box">
-     <property name="orientation">
-      <enum>Qt::Horizontal</enum>
-     </property>
-     <property name="standardButtons">
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-     </property>
-    </widget>
-   </item>
-  </layout>
- </widget>
- <customwidgets>
-  <customwidget>
-   <class>EnLineEdit</class>
-   <extends>QLineEdit</extends>
-   <header>widgets.h</header>
-  </customwidget>
-  <customwidget>
-   <class>MultiCompleteLineEdit</class>
-   <extends>QLineEdit</extends>
-   <header>calibre/gui2/complete.h</header>
-  </customwidget>
-  <customwidget>
-   <class>MultiCompleteComboBox</class>
-   <extends>QComboBox</extends>
-   <header>calibre/gui2/complete.h</header>
-  </customwidget>
-  <customwidget>
-   <class>FormatList</class>
-   <extends>QListWidget</extends>
-   <header location="global">calibre/gui2/widgets.h</header>
-  </customwidget>
-  <customwidget>
-   <class>ImageView</class>
-   <extends>QWidget</extends>
-   <header>calibre/gui2/widgets.h</header>
-   <container>1</container>
-  </customwidget>
-  <customwidget>
-   <class>Editor</class>
-   <extends>QWidget</extends>
-   <header location="global">calibre/gui2/comments_editor.h</header>
-   <container>1</container>
-  </customwidget>
- </customwidgets>
- <tabstops>
-  <tabstop>title</tabstop>
-  <tabstop>auto_title_sort</tabstop>
-  <tabstop>title_sort</tabstop>
-  <tabstop>swap_button</tabstop>
-  <tabstop>authors</tabstop>
-  <tabstop>auto_author_sort</tabstop>
-  <tabstop>author_sort</tabstop>
-  <tabstop>rating</tabstop>
-  <tabstop>publisher</tabstop>
-  <tabstop>tags</tabstop>
-  <tabstop>tag_editor_button</tabstop>
-  <tabstop>series</tabstop>
-  <tabstop>remove_series_button</tabstop>
-  <tabstop>series_index</tabstop>
-  <tabstop>isbn</tabstop>
-  <tabstop>date</tabstop>
-  <tabstop>pubdate</tabstop>
-  <tabstop>fetch_metadata_button</tabstop>
-  <tabstop>button_set_cover</tabstop>
-  <tabstop>button_set_metadata</tabstop>
-  <tabstop>formats</tabstop>
-  <tabstop>add_format_button</tabstop>
-  <tabstop>remove_format_button</tabstop>
-  <tabstop>cover_button</tabstop>
-  <tabstop>trim_cover_button</tabstop>
-  <tabstop>reset_cover</tabstop>
-  <tabstop>fetch_cover_button</tabstop>
-  <tabstop>generate_cover_button</tabstop>
-  <tabstop>button_box</tabstop>
-  <tabstop>scrollArea</tabstop>
-  <tabstop>central_widget</tabstop>
- </tabstops>
- <resources>
-  <include location="../../../../resources/images.qrc"/>
- </resources>
- <connections>
-  <connection>
-   <sender>button_box</sender>
-   <signal>accepted()</signal>
-   <receiver>MetadataSingleDialog</receiver>
-   <slot>accept()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>261</x>
-     <y>710</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>157</x>
-     <y>274</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>button_box</sender>
-   <signal>rejected()</signal>
-   <receiver>MetadataSingleDialog</receiver>
-   <slot>reject()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>329</x>
-     <y>710</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>286</x>
-     <y>274</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
-</ui>

src/calibre/gui2/metadata/bulk_download.py

@@ -1,308 +1,195 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback
-from threading import Thread
-from Queue import Queue, Empty
 from functools import partial
+from itertools import izip
+from threading import Event
 
-from PyQt4.Qt import QObject, QTimer, QDialog, \
+from PyQt4.Qt import (QIcon, QDialog,
-        QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
+        QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt)
 
-from calibre.ebooks.metadata.fetch import search, get_social_metadata
+from calibre.gui2.threaded_jobs import ThreadedJob
-from calibre.gui2 import config, error_dialog
+from calibre.ebooks.metadata.sources.identify import identify, msprefs
-from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.ebooks.metadata.sources.covers import download_cover
-from calibre.ebooks.metadata.covers import download_cover
+from calibre.ebooks.metadata.book.base import Metadata
-from calibre.customize.ui import get_isbndb_key
+from calibre.customize.ui import metadata_plugins
+from calibre.ptempfile import PersistentTemporaryFile
 
-class Worker(Thread):
+# Start download {{{
-    'Cover downloader'
+def show_config(gui, parent):
+    from calibre.gui2.preferences import show_config_widget
+    show_config_widget('Sharing', 'Metadata download', parent=parent,
+            gui=gui, never_shutdown=True)
 
-    def __init__(self):
+class ConfirmDialog(QDialog):
-        Thread.__init__(self)
-        self.daemon = True
-        self.jobs = Queue()
-        self.results = Queue()
 
-    def run(self):
+    def __init__(self, ids, parent):
-        while True:
+        QDialog.__init__(self, parent)
-            id, mi = self.jobs.get()
+        self.setWindowTitle(_('Schedule download?'))
-            if not getattr(mi, 'isbn', False):
+        self.setWindowIcon(QIcon(I('dialog_question.png')))
-                break
+
+        l = self.l = QGridLayout()
+        self.setLayout(l)
+
+        i = QLabel(self)
+        i.setPixmap(QPixmap(I('dialog_question.png')))
+        l.addWidget(i, 0, 0)
+
+        t = QLabel(
+            '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will'
+                ' run in the background. Proceed?')%len(ids) +
+            '<p>'+_('You can monitor the progress of the download '
+                'by clicking the rotating spinner in the bottom right '
+                'corner.') +
+            '<p>'+_('When the download completes you will be asked for'
+                ' confirmation before calibre applies the downloaded metadata.')
+            )
+        t.setWordWrap(True)
+        l.addWidget(t, 0, 1)
+        l.setColumnStretch(0, 1)
+        l.setColumnStretch(1, 100)
+
+        self.identify = self.covers = True
+        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
+        self.bb.rejected.connect(self.reject)
+        b = self.bb.addButton(_('Download only &metadata'),
+                self.bb.AcceptRole)
+        b.clicked.connect(self.only_metadata)
+        b.setIcon(QIcon(I('edit_input.png')))
+        b = self.bb.addButton(_('Download only &covers'),
+                self.bb.AcceptRole)
+        b.clicked.connect(self.only_covers)
+        b.setIcon(QIcon(I('default_cover.png')))
+        b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole)
+        b.setIcon(QIcon(I('config.png')))
+        b.clicked.connect(partial(show_config, parent, self))
+        l.addWidget(self.bb, 1, 0, 1, 2)
+        b = self.bb.addButton(_('Download &both'),
+                self.bb.AcceptRole)
+        b.clicked.connect(self.accept)
+        b.setDefault(True)
+        b.setAutoDefault(True)
+        b.setIcon(QIcon(I('ok.png')))
+
+        self.resize(self.sizeHint())
+        b.setFocus(Qt.OtherFocusReason)
+
+    def only_metadata(self):
+        self.covers = False
+        self.accept()
+
+    def only_covers(self):
+        self.identify = False
+        self.accept()
+
+def start_download(gui, ids, callback):
+    d = ConfirmDialog(ids, gui)
+    ret = d.exec_()
+    d.b.clicked.disconnect()
+    if ret != d.Accepted:
+        return
+
+    job = ThreadedJob('metadata bulk download',
+            _('Download metadata for %d books')%len(ids),
+            download, (ids, gui.current_db, d.identify, d.covers), {}, callback)
+    gui.job_manager.run_threaded_job(job)
+    gui.status_bar.show_message(_('Metadata download started'), 3000)
+# }}}
+
+def get_job_details(job):
+    id_map, failed_ids, failed_covers, title_map, all_failed = job.result
+    det_msg = []
+    for i in failed_ids | failed_covers:
+        title = title_map[i]
+        if i in failed_ids:
+            title += (' ' + _('(Failed metadata)'))
+        if i in failed_covers:
+            title += (' ' + _('(Failed cover)'))
+        det_msg.append(title)
+    det_msg = '\n'.join(det_msg)
+    return id_map, failed_ids, failed_covers, all_failed, det_msg
+
+def merge_result(oldmi, newmi):
+    dummy = Metadata(_('Unknown'))
+    for f in msprefs['ignore_fields']:
+        if ':' not in f:
+            setattr(newmi, f, getattr(dummy, f))
+    fields = set()
+    for plugin in metadata_plugins(['identify']):
+        fields |= plugin.touched_fields
+
+    for f in fields:
+        # Optimize so that set_metadata does not have to do extra work later
+        if not f.startswith('identifier:'):
+            if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)):
+                setattr(newmi, f, getattr(dummy, f))
+
+    newmi.last_modified = oldmi.last_modified
+
+    return newmi
+
+def download(ids, db, do_identify, covers,
+        log=None, abort=None, notifications=None):
+    ids = list(ids)
+    metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False)
+        for i in ids]
+    failed_ids = set()
+    failed_covers = set()
+    title_map = {}
+    ans = {}
+    count = 0
+    all_failed = True
+    '''
+    # Test apply dialog
+    all_failed = do_identify = covers = False
+    '''
+    for i, mi in izip(ids, metadata):
+        if abort.is_set():
+            log.error('Aborting...')
+            break
+        title, authors, identifiers = mi.title, mi.authors, mi.identifiers
+        title_map[i] = title
+        if do_identify:
+            results = []
             try:
-                cdata, errors = download_cover(mi)
+                results = identify(log, Event(), title=title, authors=authors,
-                if cdata:
+                    identifiers=identifiers)
-                    self.results.put((id, mi, True, cdata))
-                else:
-                    msg = []
-                    for e in errors:
-                        if not e[0]:
-                            msg.append(e[-1] + ' - ' + e[1])
-                    self.results.put((id, mi, False, '\n'.join(msg)))
             except:
-                self.results.put((id, mi, False, traceback.format_exc()))
-
-    def __enter__(self):
-        self.start()
-        return self
-
-    def __exit__(self, *args):
-        self.jobs.put((False, False))
-
-
-class DownloadMetadata(Thread):
-    'Metadata downloader'
-
-    def __init__(self, db, ids, get_covers, set_metadata=True,
-            get_social_metadata=True):
-        Thread.__init__(self)
-        self.daemon = True
-        self.metadata = {}
-        self.covers   = {}
-        self.set_metadata = set_metadata
-        self.get_social_metadata = get_social_metadata
-        self.social_metadata_exceptions = []
-        self.db = db
-        self.updated = set([])
-        self.get_covers = get_covers
-        self.worker = Worker()
-        self.results = Queue()
-        self.keep_going = True
-        for id in ids:
-            self.metadata[id] = db.get_metadata(id, index_is_id=True)
-            self.metadata[id].rating = None
-        self.total = len(ids)
-        if self.get_covers:
-            self.total += len(ids)
-        self.fetched_metadata = {}
-        self.fetched_covers = {}
-        self.failures = {}
-        self.cover_failures = {}
-        self.exception = self.tb = None
-
-    def run(self):
-        try:
-            self._run()
-        except Exception as e:
-            self.exception = e
-            self.tb = traceback.format_exc()
-
-    def _run(self):
-        self.key = get_isbndb_key()
-        if not self.key:
-            self.key = None
-        with self.worker:
-            for id, mi in self.metadata.items():
-                if not self.keep_going:
-                    break
-                args = {}
-                if mi.isbn:
-                    args['isbn'] = mi.isbn
-                else:
-                    if mi.is_null('title'):
-                        self.failures[id] = \
-                                _('Book has neither title nor ISBN')
-                        continue
-                    args['title'] = mi.title
-                    if mi.authors and mi.authors[0] != _('Unknown'):
-                        args['author'] = mi.authors[0]
-                if self.key:
-                    args['isbndb_key'] = self.key
-                results, exceptions = search(**args)
-                if results:
-                    fmi = results[0]
-                    self.fetched_metadata[id] = fmi
-                    if self.get_covers:
-                        if fmi.isbn:
-                            self.worker.jobs.put((id, fmi))
-                        else:
-                            self.results.put((id, 'cover', False, mi.title))
-                    if (not config['overwrite_author_title_metadata']):
-                        fmi.authors = mi.authors
-                        fmi.author_sort = mi.author_sort
-                        fmi.title = mi.title
-                    mi.smart_update(fmi)
-                    if mi.isbn and self.get_social_metadata:
-                        self.social_metadata_exceptions = get_social_metadata(mi)
-                        if mi.rating:
-                            mi.rating *= 2
-                    if not self.get_social_metadata:
-                        mi.tags = []
-                    self.results.put((id, 'metadata', True, mi.title))
-                else:
-                    self.failures[id] = _('No matches found for this book')
-                    self.results.put((id, 'metadata', False, mi.title))
-                    self.results.put((id, 'cover', False, mi.title))
-                self.commit_covers()
-
-        self.commit_covers(True)
-
-    def commit_covers(self, all=False):
-        if all:
-            self.worker.jobs.put((False, False))
-        while True:
-            try:
-                id, fmi, ok, cdata = self.worker.results.get_nowait()
-                if ok:
-                    self.fetched_covers[id] = cdata
-                    self.results.put((id, 'cover', ok, fmi.title))
-                else:
-                    self.results.put((id, 'cover', ok, fmi.title))
-                    try:
-                        self.cover_failures[id] = unicode(cdata)
-                    except:
-                        self.cover_failures[id] = repr(cdata)
-            except Empty:
-                if not all or not self.worker.is_alive():
-                    return
-
-class DoDownload(QObject):
-
-    def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
-            get_social_metadata=True):
-        QObject.__init__(self, parent)
-        self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
-        self.pd.canceled_signal.connect(self.cancel)
-        self.downloader = None
-        self.create = partial(DownloadMetadata, db, ids, get_covers,
-                set_metadata=set_metadata,
-                get_social_metadata=get_social_metadata)
-        self.get_covers = get_covers
-        self.db = db
-        self.updated = set([])
-        self.total = len(ids)
-        self.keep_going = True
-
-    def exec_(self):
-        QTimer.singleShot(50, self.do_one)
-        ret = self.pd.exec_()
-        if getattr(self.downloader, 'exception', None) is not None and \
-                ret == self.pd.Accepted:
-            error_dialog(self.parent(), _('Failed'),
-                    _('Failed to download metadata'), show=True)
-        else:
-            self.show_report()
-        return ret
-
-    def cancel(self, *args):
-        self.keep_going = False
-        self.downloader.keep_going = False
-        self.pd.reject()
-
-    def do_one(self):
-        try:
-            if not self.keep_going:
-                return
-            if self.downloader is None:
-                self.downloader = self.create()
-                self.downloader.start()
-                self.pd.set_min(0)
-                self.pd.set_max(self.downloader.total)
-            try:
-                r = self.downloader.results.get_nowait()
-                self.handle_result(r)
-            except Empty:
                 pass
-            if not self.downloader.is_alive():
+            if results:
-                while True:
+                all_failed = False
-                    try:
+                mi = merge_result(mi, results[0])
-                        r = self.downloader.results.get_nowait()
+                identifiers = mi.identifiers
-                        self.handle_result(r)
+                if not mi.is_null('rating'):
-                    except Empty:
+                    # set_metadata expects a rating out of 10
-                        break
+                    mi.rating *= 2
-                self.pd.accept()
-                return
-        except:
-            self.cancel()
-            raise
-        QTimer.singleShot(50, self.do_one)
-
-    def handle_result(self, r):
-        id_, typ, ok, title = r
-        what = _('cover') if typ == 'cover' else _('metadata')
-        which = _('Downloaded') if ok else _('Failed to get')
-        if self.get_covers or typ != 'cover' or ok:
-            # Do not show message when cover fetch fails if user didn't ask to
-            # download covers
-            self.pd.set_msg(_('%s %s for: %s') % (which, what, title))
-        self.pd.value += 1
-        if ok:
-            self.updated.add(id_)
-            if typ == 'cover':
-                try:
-                    self.db.set_cover(id_,
-                            self.downloader.fetched_covers.pop(id_))
-                except:
-                    self.downloader.cover_failures[id_] = \
-                            traceback.format_exc()
             else:
-                try:
+                log.error('Failed to download metadata for', title)
-                    self.set_metadata(id_)
+                failed_ids.add(i)
-                except:
+                # We don't want set_metadata operating on anything but covers
-                    self.downloader.failures[id_] = \
+                mi = merge_result(mi, mi)
-                            traceback.format_exc()
+        if covers:
-
+            cdata = download_cover(log, title=title, authors=authors,
-    def set_metadata(self, id_):
+                    identifiers=identifiers)
-        mi = self.downloader.metadata[id_]
+            if cdata is not None:
-        if self.downloader.set_metadata:
+                with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
-            self.db.set_metadata(id_, mi)
+                    f.write(cdata[-1])
-        if not self.downloader.set_metadata and self.downloader.get_social_metadata:
+                    mi.cover = f.name
-            if mi.rating:
+                all_failed = False
-                self.db.set_rating(id_, mi.rating)
+            else:
-            if mi.tags:
+                failed_covers.add(i)
-                self.db.set_tags(id_, mi.tags)
+        ans[i] = mi
-            if mi.comments:
+        count += 1
-                self.db.set_comment(id_, mi.comments)
+        notifications.put((count/len(ids),
-            if mi.series:
+            _('Downloaded %d of %d')%(count, len(ids))))
-                self.db.set_series(id_, mi.series)
+    log('Download complete, with %d failures'%len(failed_ids))
-                if mi.series_index is not None:
+    return (ans, failed_ids, failed_covers, title_map, all_failed)
-                    self.db.set_series_index(id_, mi.series_index)
-
-    def show_report(self):
-        f, cf = self.downloader.failures, self.downloader.cover_failures
-        report = []
-        if f:
-            report.append(
-                '<h3>Failed to download metadata for the following:</h3><ol>')
-            for id_, err in f.items():
-                mi = self.downloader.metadata[id_]
-                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
-                    unicode(err)))
-            report.append('</ol>')
-        if cf:
-            report.append(
-                '<h3>Failed to download cover for the following:</h3><ol>')
-            for id_, err in cf.items():
-                mi = self.downloader.metadata[id_]
-                report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
-                    unicode(err)))
-            report.append('</ol>')
-
-        if len(self.updated) != self.total or report:
-            d = QDialog(self.parent())
-            bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d)
-            v1 = QVBoxLayout()
-            d.setLayout(v1)
-            d.setWindowTitle(_('Done'))
-            v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') %
-                (len(self.updated), self.total)))
-            gb = QGroupBox(_('Details'), self.parent())
-            v2 = QVBoxLayout()
-            gb.setLayout(v2)
-            b = QTextBrowser(self.parent())
-            v2.addWidget(b)
-            b.setHtml('\n'.join(report))
-            v1.addWidget(gb)
-            v1.addWidget(bb)
-            bb.accepted.connect(d.accept)
-            d.resize(800, 600)
-            d.exec_()
-
 
 
 

src/calibre/gui2/metadata/bulk_download2.py

@@ -1,195 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-from functools import partial
-from itertools import izip
-from threading import Event
-
-from PyQt4.Qt import (QIcon, QDialog,
-        QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt)
-
-from calibre.gui2.threaded_jobs import ThreadedJob
-from calibre.ebooks.metadata.sources.identify import identify, msprefs
-from calibre.ebooks.metadata.sources.covers import download_cover
-from calibre.ebooks.metadata.book.base import Metadata
-from calibre.customize.ui import metadata_plugins
-from calibre.ptempfile import PersistentTemporaryFile
-
-# Start download {{{
-def show_config(gui, parent):
-    from calibre.gui2.preferences import show_config_widget
-    show_config_widget('Sharing', 'Metadata download', parent=parent,
-            gui=gui, never_shutdown=True)
-
-class ConfirmDialog(QDialog):
-
-    def __init__(self, ids, parent):
-        QDialog.__init__(self, parent)
-        self.setWindowTitle(_('Schedule download?'))
-        self.setWindowIcon(QIcon(I('dialog_question.png')))
-
-        l = self.l = QGridLayout()
-        self.setLayout(l)
-
-        i = QLabel(self)
-        i.setPixmap(QPixmap(I('dialog_question.png')))
-        l.addWidget(i, 0, 0)
-
-        t = QLabel(
-            '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will'
-                ' run in the background. Proceed?')%len(ids) +
-            '<p>'+_('You can monitor the progress of the download '
-                'by clicking the rotating spinner in the bottom right '
-                'corner.') +
-            '<p>'+_('When the download completes you will be asked for'
-                ' confirmation before calibre applies the downloaded metadata.')
-            )
-        t.setWordWrap(True)
-        l.addWidget(t, 0, 1)
-        l.setColumnStretch(0, 1)
-        l.setColumnStretch(1, 100)
-
-        self.identify = self.covers = True
-        self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
-        self.bb.rejected.connect(self.reject)
-        b = self.bb.addButton(_('Download only &metadata'),
-                self.bb.AcceptRole)
-        b.clicked.connect(self.only_metadata)
-        b.setIcon(QIcon(I('edit_input.png')))
-        b = self.bb.addButton(_('Download only &covers'),
-                self.bb.AcceptRole)
-        b.clicked.connect(self.only_covers)
-        b.setIcon(QIcon(I('default_cover.png')))
-        b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole)
-        b.setIcon(QIcon(I('config.png')))
-        b.clicked.connect(partial(show_config, parent, self))
-        l.addWidget(self.bb, 1, 0, 1, 2)
-        b = self.bb.addButton(_('Download &both'),
-                self.bb.AcceptRole)
-        b.clicked.connect(self.accept)
-        b.setDefault(True)
-        b.setAutoDefault(True)
-        b.setIcon(QIcon(I('ok.png')))
-
-        self.resize(self.sizeHint())
-        b.setFocus(Qt.OtherFocusReason)
-
-    def only_metadata(self):
-        self.covers = False
-        self.accept()
-
-    def only_covers(self):
-        self.identify = False
-        self.accept()
-
-def start_download(gui, ids, callback):
-    d = ConfirmDialog(ids, gui)
-    ret = d.exec_()
-    d.b.clicked.disconnect()
-    if ret != d.Accepted:
-        return
-
-    job = ThreadedJob('metadata bulk download',
-            _('Download metadata for %d books')%len(ids),
-            download, (ids, gui.current_db, d.identify, d.covers), {}, callback)
-    gui.job_manager.run_threaded_job(job)
-    gui.status_bar.show_message(_('Metadata download started'), 3000)
-# }}}
-
-def get_job_details(job):
-    id_map, failed_ids, failed_covers, title_map, all_failed = job.result
-    det_msg = []
-    for i in failed_ids | failed_covers:
-        title = title_map[i]
-        if i in failed_ids:
-            title += (' ' + _('(Failed metadata)'))
-        if i in failed_covers:
-            title += (' ' + _('(Failed cover)'))
-        det_msg.append(title)
-    det_msg = '\n'.join(det_msg)
-    return id_map, failed_ids, failed_covers, all_failed, det_msg
-
-def merge_result(oldmi, newmi):
-    dummy = Metadata(_('Unknown'))
-    for f in msprefs['ignore_fields']:
-        if ':' not in f:
-            setattr(newmi, f, getattr(dummy, f))
-    fields = set()
-    for plugin in metadata_plugins(['identify']):
-        fields |= plugin.touched_fields
-
-    for f in fields:
-        # Optimize so that set_metadata does not have to do extra work later
-        if not f.startswith('identifier:'):
-            if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)):
-                setattr(newmi, f, getattr(dummy, f))
-
-    newmi.last_modified = oldmi.last_modified
-
-    return newmi
-
-def download(ids, db, do_identify, covers,
-        log=None, abort=None, notifications=None):
-    ids = list(ids)
-    metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False)
-        for i in ids]
-    failed_ids = set()
-    failed_covers = set()
-    title_map = {}
-    ans = {}
-    count = 0
-    all_failed = True
-    '''
-    # Test apply dialog
-    all_failed = do_identify = covers = False
-    '''
-    for i, mi in izip(ids, metadata):
-        if abort.is_set():
-            log.error('Aborting...')
-            break
-        title, authors, identifiers = mi.title, mi.authors, mi.identifiers
-        title_map[i] = title
-        if do_identify:
-            results = []
-            try:
-                results = identify(log, Event(), title=title, authors=authors,
-                    identifiers=identifiers)
-            except:
-                pass
-            if results:
-                all_failed = False
-                mi = merge_result(mi, results[0])
-                identifiers = mi.identifiers
-                if not mi.is_null('rating'):
-                    # set_metadata expects a rating out of 10
-                    mi.rating *= 2
-            else:
-                log.error('Failed to download metadata for', title)
-                failed_ids.add(i)
-                # We don't want set_metadata operating on anything but covers
-                mi = merge_result(mi, mi)
-        if covers:
-            cdata = download_cover(log, title=title, authors=authors,
-                    identifiers=identifiers)
-            if cdata is not None:
-                with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
-                    f.write(cdata[-1])
-                    mi.cover = f.name
-                all_failed = False
-            else:
-                failed_covers.add(i)
-        ans[i] = mi
-        count += 1
-        notifications.put((count/len(ids),
-            _('Downloaded %d of %d')%(count, len(ids))))
-    log('Download complete, with %d failures'%len(failed_ids))
-    return (ans, failed_ids, failed_covers, title_map, all_failed)
-
-
-

src/calibre/gui2/preferences/behavior.py

@@ -19,7 +19,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.oeb.iterator import is_supported
 from calibre.constants import iswindows
 from calibre.utils.icu import sort_key
-from calibre.utils.config import test_eight_code
 
 class OutputFormatSetting(Setting):
 
@@ -40,12 +39,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         r('network_timeout', prefs)
 
-
-        r('overwrite_author_title_metadata', config)
-        r('get_social_metadata', config)
-        if test_eight_code:
-            self.opt_overwrite_author_title_metadata.setVisible(False)
-            self.opt_get_social_metadata.setVisible(False)
         r('new_version_notification', config)
         r('upload_news_to_device', config)
         r('delete_news_from_library_on_upload', config)
@@ -67,13 +60,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             signal.connect(self.internally_viewed_formats_changed)
 
         r('bools_are_tristate', db.prefs, restart_required=True)
-        if test_eight_code:
+        r = self.register
-            r = self.register
+        choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
-            choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
+        r('edit_metadata_single_layout', gprefs, choices=choices)
-            r('edit_metadata_single_layout', gprefs, choices=choices)
-        else:
-            self.opt_edit_metadata_single_layout.setVisible(False)
-            self.edit_metadata_single_label.setVisible(False)
 
     def initialize(self):
         ConfigWidgetBase.initialize(self)

src/calibre/gui2/preferences/behavior.ui

@@ -14,41 +14,14 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="1">
+   <item row="1" column="0">
-    <spacer>
-     <property name="orientation">
-      <enum>Qt::Horizontal</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>10</width>
-       <height>0</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item row="0" column="0">
-    <widget class="QCheckBox" name="opt_overwrite_author_title_metadata">
-     <property name="text">
-      <string>&amp;Overwrite author and title by default when fetching metadata</string>
-     </property>
-    </widget>
-   </item>
-   <item row="0" column="2">
-    <widget class="QCheckBox" name="opt_get_social_metadata">
-     <property name="text">
-      <string>Download &amp;social metadata (tags/ratings/etc.) by default</string>
-     </property>
-    </widget>
-   </item>
-   <item row="2" column="0">
     <widget class="QCheckBox" name="opt_new_version_notification">
      <property name="text">
       <string>Show notification when &amp;new version is available</string>
      </property>
     </widget>
    </item>
-   <item row="2" column="2">
+   <item row="1" column="1">
     <widget class="QCheckBox" name="opt_bools_are_tristate">
      <property name="toolTip">
       <string>If checked, Yes/No custom columns values can be Yes, No, or Unknown.
@@ -59,21 +32,21 @@ If not checked, the values can be Yes or No.</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="3" column="0">
     <widget class="QCheckBox" name="opt_upload_news_to_device">
      <property name="text">
       <string>Automatically send downloaded &amp;news to ebook reader</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="2">
+   <item row="3" column="1">
     <widget class="QCheckBox" name="opt_delete_news_from_library_on_upload">
      <property name="text">
       <string>&amp;Delete news from library when it is automatically sent to reader</string>
      </property>
     </widget>
    </item>
-   <item row="6" column="0">
+   <item row="5" column="0">
     <layout class="QHBoxLayout">
      <item>
       <widget class="QLabel" name="label_23">
@@ -97,7 +70,7 @@ If not checked, the values can be Yes or No.</string>
      </item>
     </layout>
    </item>
-   <item row="6" column="2">
+   <item row="5" column="1">
     <layout class="QHBoxLayout">
      <item>
       <widget class="QLabel" name="label_2">
@@ -130,7 +103,7 @@ If not checked, the values can be Yes or No.</string>
      </item>
     </layout>
    </item>
-   <item row="8" column="0">
+   <item row="7" column="0">
     <layout class="QHBoxLayout">
      <item>
       <widget class="QLabel" name="priority_label">
@@ -169,7 +142,7 @@ If not checked, the values can be Yes or No.</string>
      </item>
     </layout>
    </item>
-   <item row="8" column="2">
+   <item row="7" column="1">
     <layout class="QHBoxLayout">
      <item>
       <widget class="QLabel" name="label_170">
@@ -202,7 +175,7 @@ If not checked, the values can be Yes or No.</string>
      </item>
     </layout>
    </item>
-   <item row="9" column="0">
+   <item row="8" column="0">
     <layout class="QHBoxLayout">
      <item>
       <widget class="QLabel" name="edit_metadata_single_label">
@@ -223,7 +196,7 @@ If not checked, the values can be Yes or No.</string>
      </item>
     </layout>
    </item>
-   <item row="20" column="0">
+   <item row="19" column="0">
     <widget class="QGroupBox" name="groupBox_5">
      <property name="title">
       <string>Preferred &amp;input format order:</string>
@@ -285,7 +258,7 @@ If not checked, the values can be Yes or No.</string>
      </layout>
     </widget>
    </item>
-   <item row="20" column="2">
+   <item row="19" column="1">
     <widget class="QGroupBox" name="groupBox_3">
      <property name="title">
       <string>Use internal &amp;viewer for:</string>
@@ -304,7 +277,7 @@ If not checked, the values can be Yes or No.</string>
      </layout>
     </widget>
    </item>
-   <item row="9" column="2">
+   <item row="8" column="1">
     <widget class="QPushButton" name="reset_confirmation_button">
      <property name="text">
       <string>Reset all disabled &amp;confirmation dialogs</string>

src/calibre/gui2/preferences/social.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
-
-__license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-__docformat__ = 'restructuredtext en'
-
-import time
-from threading import Thread
-
-from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \
-        QTimer
-
-from calibre.ebooks.metadata import MetaInformation
-
-class Worker(Thread):
-
-    def __init__(self, mi):
-        Thread.__init__(self)
-        self.daemon = True
-        self.mi = MetaInformation(mi)
-        self.exceptions = []
-
-    def run(self):
-        from calibre.ebooks.metadata.fetch import get_social_metadata
-        self.exceptions = get_social_metadata(self.mi)
-
-class SocialMetadata(QDialog):
-
-    TIMEOUT = 300 # seconds
-
-    def __init__(self, mi, parent):
-        QDialog.__init__(self, parent)
-
-        self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self)
-        self.mi = mi
-        self.layout = QVBoxLayout(self)
-        self.label = QLabel(_('Downloading social metadata, please wait...'), self)
-        self.label.setWordWrap(True)
-        self.layout.addWidget(self.label)
-        self.layout.addWidget(self.bbox)
-
-        self.worker = Worker(mi)
-        self.bbox.rejected.connect(self.reject)
-        self.worker.start()
-        self.start_time = time.time()
-        self.timed_out = False
-        self.rejected = False
-        QTimer.singleShot(50, self.update)
-
-    def reject(self):
-        self.rejected = True
-        QDialog.reject(self)
-
-    def update(self):
-        if self.rejected:
-            return
-        if time.time() - self.start_time > self.TIMEOUT:
-            self.timed_out = True
-            self.reject()
-            return
-        if not self.worker.is_alive():
-            self.accept()
-            return
-        QTimer.singleShot(50, self.update)
-
-    def accept(self):
-        self.mi.tags = self.worker.mi.tags
-        self.mi.rating = self.worker.mi.rating
-        self.mi.comments = self.worker.mi.comments
-        if self.worker.mi.series:
-            self.mi.series = self.worker.mi.series
-            self.mi.series_index = self.worker.mi.series_index
-        QDialog.accept(self)
-
-    @property
-    def exceptions(self):
-        return self.worker.exceptions

src/calibre/gui2/store/foyles_uk_plugin.py

@@ -57,7 +57,7 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin):
                 cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src'))
                 if cover_url:
                     cover_url = 'http://www.foyles.co.uk' + cover_url
-                print(cover_url)
+                #print(cover_url)
 
                 title = ''.join(data.xpath('.//a[@class="Title"]/text()'))
                 author = ', '.join(data.xpath('.//span[@class="Author"]/text()'))

src/calibre/library/database2.py

@@ -3217,7 +3217,6 @@ books_series_link      feeds
             if callable(callback):
                 if callback(''):
                     break
-
         return duplicates
 
     def add_custom_book_data(self, book_id, name, val):
@@ -3226,12 +3225,19 @@ books_series_link      feeds
             raise ValueError('add_custom_book_data: no such book_id %d'%book_id)
         # Do the json encode first, in case it throws an exception
         s = json.dumps(val, default=to_json)
-        self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
+        self.conn.execute('''INSERT OR REPLACE INTO books_plugin_data(book, name, val)
-                          (book_id, name))
-        self.conn.execute('''INSERT INTO books_plugin_data(book, name, val)
                              VALUES(?, ?, ?)''', (book_id, name, s))
         self.commit()
 
+    def add_multiple_custom_book_data(self, name, vals, delete_first=False):
+        if delete_first:
+            self.conn.execute('DELETE FROM books_plugin_data WHERE name=?', (name, ))
+        self.conn.executemany(
+            'INSERT OR REPLACE INTO books_plugin_data (book, name, val) VALUES (?, ?, ?)',
+            [(book_id, name, json.dumps(val, default=to_json))
+                    for book_id, val in vals.iteritems()])
+        self.commit()
+
     def get_custom_book_data(self, book_id, name, default=None):
         try:
             s = self.conn.get('''select val FROM books_plugin_data
@@ -3243,11 +3249,29 @@ books_series_link      feeds
             pass
         return default
 
+    def get_all_custom_book_data(self, name, default=None):
+        try:
+            s = self.conn.get('''select book, val FROM books_plugin_data
+                    WHERE name=?''', (name,))
+            if s is None:
+                return default
+            res = {}
+            for r in s:
+                res[r[0]] = json.loads(r[1], object_hook=from_json)
+            return res
+        except:
+            pass
+        return default
+
     def delete_custom_book_data(self, book_id, name):
         self.conn.execute('DELETE FROM books_plugin_data WHERE book=? AND name=?',
                           (book_id, name))
         self.commit()
 
+    def delete_all_custom_book_data(self, name):
+        self.conn.execute('DELETE FROM books_plugin_data WHERE name=?', (name, ))
+        self.commit()
+
     def get_ids_for_custom_book_data(self, name):
         s = self.conn.get('''SELECT book FROM books_plugin_data WHERE name=?''', (name,))
         return [x[0] for x in s]

src/calibre/manual/faq.rst

@@ -468,6 +468,18 @@ If it still wont launch, start a command prompt (press the windows key and R; th
 
 Post any output you see in a help message on the `Forum <http://www.mobileread.com/forums/forumdisplay.php?f=166>`_.
 
+|app| freeze when I click on anything?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are three possible things I know of, that can cause this:
+
+    * You recently connected an external monitor or TV to your computer. In this case, whenever |app| opens a new window like the edit metadata window or the conversion dialog, it appears on the second monitor where you dont notice it and so you think |app| has frozen. Disconnect your second monitor and restart calibre.
+
+    * You are using a Wacom branded mouse. There is an incompatibility between Wacom mice and the graphics toolkit |app| uses. Try using a non-Wacom mouse.
+
+    * You have invalid files in your fonts folder. If this is the case, start |app| in debug mode as desribed in the previous answer and you will get messages about invalid files in :file:`C:\\Windows\\fonts`. Delete these files and you will be fine.
+
+
 |app| is not starting on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~