GwR wip

resources/recipes/hindu.recipe

@@ -2,7 +2,7 @@ from __future__ import with_statement
 __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 
-import re
+import time
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class TheHindu(BasicNewsRecipe):
@@ -10,45 +10,41 @@ class TheHindu(BasicNewsRecipe):
     language = 'en_IN'
 
     oldest_article        = 7
-    __author__            = 'Kovid Goyal and Sujata Raman'
+    __author__            = 'Kovid Goyal'
     max_articles_per_feed = 100
     no_stylesheets = True
 
-    remove_tags_before = {'name':'font', 'class':'storyhead'}
+    keep_only_tags = [dict(id='content')]
-    preprocess_regexps = [
+    remove_tags = [dict(attrs={'class':['article-links', 'breadcr']}),
-                (re.compile(r'<!-- story ends -->.*', re.DOTALL),
+            dict(id=['email-section', 'right-column', 'printfooter'])]
-                 lambda match: '</body></html>'),
+
-                          ]
+    extra_css = '.photo-caption { font-size: smaller }'
-    extra_css = '''
-                .storyhead{font-family:Arial,Helvetica,sans-serif; font-size:large; color:#000099;}
-                body{font-family:Verdana,Arial,Helvetica,sans-serif; font-size:x-small; text-align:left;}
-                '''
-    feeds          = [
-      (u'Main - Front Page', u'http://www.hindu.com/rss/01hdline.xml'),
-      (u'Main - National', u'http://www.hindu.com/rss/02hdline.xml'),
-      (u'Main - International', u'http://www.hindu.com/rss/03hdline.xml'),
-      (u'Main - Opinion', u'http://www.hindu.com/rss/05hdline.xml'),
-      (u'Main - Business', u'http://www.hindu.com/rss/06hdline.xml'),
-      (u'Main - Sport', u'http://www.hindu.com/rss/07hdline.xml'),
-      (u'Main - Weather / Religion / Crossword / Cartoon',
-       u'http://www.hindu.com/rss/10hdline.xml'),
-      (u'Main - Engagements', u'http://www.hindu.com/rss/26hdline.xml'),
-      (u'Supplement - Literary Review',
-       u'http://www.hindu.com/rss/lrhdline.xml'),
-      (u'Supplement - Sunday Magazine',
-       u'http://www.hindu.com/rss/maghdline.xml'),
-      (u'Supplement - Open Page', u'http://www.hindu.com/rss/ophdline.xml'),
-      (u'Supplement - Business Review',
-       u'http://www.hindu.com/rss/bizhdline.xml'),
-      (u'Supplement - Book Review',
-       u'http://www.hindu.com/rss/brhdline.xml'),
-      (u'Supplement - Science & Technology',
-       u'http://www.hindu.com/rss/setahdline.xml')
-      ]
 
     def postprocess_html(self, soup, first_fetch):
         for t in soup.findAll(['table', 'tr', 'td','center']):
             t.name = 'div'
-
-
         return soup
+
+    def parse_index(self):
+        today = time.strftime('%Y-%m-%d')
+        soup = self.index_to_soup(
+                'http://www.thehindu.com/todays-paper/tp-index/?date=' + today)
+        div = soup.find(id='left-column')
+        feeds = []
+        current_section = None
+        current_articles = []
+        for x in div.findAll(['h3', 'div']):
+            if current_section and x.get('class', '') == 'tpaper':
+                a = x.find('a', href=True)
+                if a is not None:
+                    current_articles.append({'url':a['href']+'?css=print',
+                        'title':self.tag_to_string(a), 'date': '',
+                        'description':''})
+            if x.name == 'h3':
+                if current_section and current_articles:
+                    feeds.append((current_section, current_articles))
+                current_section = self.tag_to_string(x)
+                current_articles = []
+        return feeds
+
+

resources/recipes/toi.recipe

@@ -1,21 +1,16 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
 
 class TimesOfIndia(BasicNewsRecipe):
     title          = u'Times of India'
     language       = 'en_IN'
-    __author__     = 'Krittika Goyal'
+    __author__     = 'Kovid Goyal'
     oldest_article = 1 #days
     max_articles_per_feed = 25
 
-    remove_stylesheets = True
+    no_stylesheets = True
+    keep_only_tags = [dict(attrs={'class':'prttabl'})]
     remove_tags = [
-       dict(name='iframe'),
+            dict(style=lambda x: x and 'float' in x)
-       dict(name='td', attrs={'class':'newptool1'}),
-       dict(name='div', attrs={'id':'newptool'}),
-       dict(name='ul', attrs={'class':'newtabcontent_tabs_new'}),
-       dict(name='b', text='Topics'),
-       dict(name='span', text=':'),
     ]
 
     feeds          = [
@@ -42,13 +37,8 @@ class TimesOfIndia(BasicNewsRecipe):
 ('Most Read',
  'http://timesofindia.indiatimes.com/rssfeedmostread.cms')
 ]
+    def print_version(self, url):
+        return url + '?prtpage=1'
 
     def preprocess_html(self, soup):
-        heading = soup.find(name='h1', attrs={'class':'heading'})
-        td = heading.findParent(name='td')
-        td.extract()
-        soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
-        body = soup.find(name='body')
-        body.insert(0, td)
-        td.name = 'div'
         return soup

resources/recipes/winnipeg_sun.recipe

@@ -0,0 +1,35 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277647803(BasicNewsRecipe):
+    title          = u'Winnipeg Sun'
+    __author__            = 'rty'
+    __version__            = '1.0'
+    oldest_article = 2
+    pubisher  = 'www.winnipegsun.com'
+    description           = 'Winnipeg Newspaper'
+    category              = 'News, Winnipeg, Canada'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'UTF-8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    language = 'en_CA'
+    feeds          = [
+	(u'News', u'http://www.winnipegsun.com/news/rss.xml'),
+	(u'Columnists', u'http://www.winnipegsun.com/columnists/rss.xml'),
+	(u'Editorial', u'http://www.winnipegsun.com/comment/editorial/rss.xml'),
+	(u'Entertainments', u'http://www.winnipegsun.com/entertainment/rss.xml'),
+	(u'Life', u'http://www.winnipegsun.com/life/rss.xml'),
+	(u'Money', u'http://www.winnipegsun.com/money/rss.xml')
+	]
+    keep_only_tags = [
+                              dict(name='div', attrs={'id':'article'}),
+                              ]
+    remove_tags = [
+           	dict(name='div', attrs={'class':['leftBox','bottomBox clear']}),
+    	dict(name='ul', attrs={'class':'tabs dl contentSwap'}),
+           	dict(name='div', attrs={'id':'commentsBottom'}),
+                      ]
+    remove_tags_after = [
+                             dict(name='div', attrs={'class':'bottomBox clear'})
+                           ]

src/calibre/customize/builtins.py

@@ -453,7 +453,7 @@ from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE, AVANT
+from calibre.devices.misc import PALMPRE, AVANT, SWEEX
 from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG
 from calibre.devices.kobo.driver import KOBO
 
@@ -499,7 +499,6 @@ plugins += [
 ]
 # Order here matters. The first matched device is the one used.
 plugins += [
-    ITUNES,
     HANLINV3,
     HANLINV5,
     BLACKBERRY,
@@ -551,6 +550,8 @@ plugins += [
     FOLDER_DEVICE_FOR_CONFIG,
     AVANT,
     MENTOR,
+    SWEEX,
+    ITUNES,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/customize/profiles.py

@@ -279,6 +279,7 @@ class iPadOutput(OutputProfile):
                 width:18%;
 				}
             .article_link {
+            	color: #593f29;
                 font-style: italic;
                 }
             .article_next {
@@ -310,7 +311,7 @@ class iPadOutput(OutputProfile):
 				}
 
             .touchscreen_navbar {
-                background:#ccc;
+                background:#c3bab2;
                 border:#ccc 0px solid;
                 border-collapse:separate;
                 border-spacing:1px;
@@ -322,11 +323,17 @@ class iPadOutput(OutputProfile):
             .touchscreen_navbar td {
                 background:#fff;
                 font-family:Helvetica;
-                font-size:90%;
+                font-size:80%;
-                padding: 5px;
+                /* UI touchboxes use 8px padding */
+                padding: 6px;
                 text-align:center;
                 }
 
+			.touchscreen_navbar td a:link {
+				color: #593f29;
+				text-decoration: none;
+				}
+
 			/* Index formatting */
 			.publish_date {
 				text-align:center;

src/calibre/devices/android/driver.py

@@ -51,8 +51,8 @@ class ANDROID(USBMS):
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD',
-            'PR OD_GT-I9000', 'FILE-STOR_GADGET']
+            'GT-I9000', 'FILE-STOR_GADGET']
-    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PR OD_GT-I9000_CARD',
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD',
             'FILE-STOR_GADGET']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'

src/calibre/devices/apple/driver.py

@@ -38,6 +38,7 @@ if iswindows:
 class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
+    #SUPPORTS_SUB_DIRS = True
 
     @classmethod
     def _config_base_name(cls):
@@ -87,7 +88,7 @@ class ITUNES(DriverBase):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (0,8,0)
+    version        = (0,9,0)
 
     OPEN_FEEDBACK_MESSAGE = _(
         'Apple device detected, launching iTunes, please wait ...')
@@ -106,53 +107,55 @@ class ITUNES(DriverBase):
     BCD = [0x01]
 
     # iTunes enumerations
-    Sources = [
+    Audiobooks = [
-                'Unknown',
+        'Audible file',
-                'Library',
+        'MPEG audio file',
-                'iPod',
+        'Protected AAC audio file'
-                'AudioCD',
+        ]
-                'MP3CD',
-                'Device',
-                'RadioTuner',
-                'SharedLibrary']
-
     ArtworkFormat = [
-                'Unknown',
+        'Unknown',
-                'JPEG',
+        'JPEG',
-                'PNG',
+        'PNG',
-                'BMP'
+        'BMP'
-                ]
+        ]
-
     PlaylistKind = [
-                'Unknown',
+        'Unknown',
-                'Library',
+        'Library',
-                'User',
+        'User',
-                'CD',
+        'CD',
-                'Device',
+        'Device',
-                'Radio Tuner'
+        'Radio Tuner'
-                ]
+        ]
-
     PlaylistSpecialKind = [
-                'Unknown',
+        'Unknown',
-                'Purchased Music',
+        'Purchased Music',
-                'Party Shuffle',
+        'Party Shuffle',
-                'Podcasts',
+        'Podcasts',
-                'Folder',
+        'Folder',
-                'Video',
+        'Video',
-                'Music',
+        'Music',
-                'Movies',
+        'Movies',
-                'TV Shows',
+        'TV Shows',
-                'Books',
+        'Books',
-                ]
+        ]
-
     SearchField = [
-                'All',
+        'All',
-                'Visible',
+        'Visible',
-                'Artists',
+        'Artists',
-                'Albums',
+        'Albums',
-                'Composers',
+        'Composers',
-                'SongNames',
+        'SongNames',
-                ]
+        ]
+    Sources = [
+        'Unknown',
+        'Library',
+        'iPod',
+        'AudioCD',
+        'MP3CD',
+        'Device',
+        'RadioTuner',
+        'SharedLibrary'
+        ]
 
     # Cover art size limits
     MAX_COVER_WIDTH = 510
@@ -532,8 +535,11 @@ class ITUNES(DriverBase):
         # Turn off the Save template
         cw.opt_save_template.setVisible(False)
         cw.label.setVisible(False)
-        # Repurpose the checkbox
+        # Repurpose the metadata checkbox
         cw.opt_read_metadata.setText(_("Use Series as Category in iTunes/iBooks"))
+        # Repurpose the use_subdirs checkbox
+#         cw.opt_use_subdirs.setText(_("Do not display books in iTunes/iBooks database\n"
+#                                      "(shortens load time with very large collections)."))
         return cw
 
     def delete_books(self, paths, end_session=True):
@@ -1766,7 +1772,7 @@ class ITUNES(DriverBase):
 
                 for book in books:
                     # This may need additional entries for international iTunes users
-                    if book.kind() in ['MPEG audio file']:
+                    if book.kind() in self.Audiobooks:
                         if DEBUG:
                             self.log.info("   ignoring '%s' of type '%s'" % (book.name(), book.kind()))
                     else:
@@ -1798,7 +1804,7 @@ class ITUNES(DriverBase):
 
                     for book in dev_books:
                         # This may need additional entries for international iTunes users
-                        if book.KindAsString in ['MPEG audio file']:
+                        if book.KindAsString in self.Audiobooks:
                             if DEBUG:
                                 self.log.info("   ignoring '%s' of type '%s'" % (book.Name, book.KindAsString))
                         else:
@@ -1899,7 +1905,7 @@ class ITUNES(DriverBase):
                     lib_books = pl.file_tracks()
                     for book in lib_books:
                         # This may need additional entries for international iTunes users
-                        if book.kind() in ['MPEG audio file']:
+                        if book.kind() in self.Audiobooks:
                             if DEBUG:
                                 self.log.info("   ignoring '%s' of type '%s'" % (book.name(), book.kind()))
                         else:
@@ -1955,7 +1961,7 @@ class ITUNES(DriverBase):
                 try:
                     for book in lib_books:
                         # This may need additional entries for international iTunes users
-                        if book.KindAsString in ['MPEG audio file']:
+                        if book.KindAsString in self.Audiobooks:
                             if DEBUG:
                                 self.log.info("   ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString))
                         else:

src/calibre/devices/kobo/books.py

@@ -13,7 +13,7 @@ from calibre import isbytestring
 
 class Book(MetaInformation):
 
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections']
+    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
 
     JSON_ATTRS = [
         'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
@@ -27,6 +27,7 @@ class Book(MetaInformation):
 
         MetaInformation.__init__(self, '')
         self.device_collections = []
+        self._new_book = False
 
         self.path = os.path.join(prefix, lpath)
         if os.sep == '\\':

src/calibre/devices/misc.py

@@ -49,3 +49,23 @@ class AVANT(USBMS):
     EBOOK_DIR_MAIN = ''
     SUPPORTS_SUB_DIRS = True
 
+class SWEEX(USBMS):
+    name           = 'Sweex Device Interface'
+    gui_name       = 'Sweex'
+    description    = _('Communicate with the Sweex MM300')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'prc', 'fb2', 'html', 'rtf', 'chm', 'pdf', 'txt']
+
+    VENDOR_ID   = [0x0525]
+    PRODUCT_ID  = [0xa4a5]
+    BCD         = [0x0319]
+
+    VENDOR_NAME = 'SWEEX'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'EBOOKREADER'
+
+    EBOOK_DIR_MAIN = ''
+    SUPPORTS_SUB_DIRS = True
+

src/calibre/devices/usbms/books.py

@@ -15,7 +15,7 @@ from calibre.utils.config import prefs
 
 class Book(MetaInformation):
 
-    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections']
+    BOOK_ATTRS = ['lpath', 'size', 'mime', 'device_collections', '_new_book']
 
     JSON_ATTRS = [
         'lpath', 'title', 'authors', 'mime', 'size', 'tags', 'author_sort',
@@ -30,6 +30,7 @@ class Book(MetaInformation):
 
         MetaInformation.__init__(self, '')
 
+        self._new_book = False
         self.device_collections = []
         self.path = os.path.join(prefix, lpath)
         if os.sep == '\\':
@@ -133,12 +134,21 @@ class CollectionsBookList(BookList):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
-        collection_attributes = list(collection_attributes)
+        for book in self:
-        if prefs['preserve_user_collections']:
+            # The default: leave the book in all existing collections. Do not
-            collection_attributes += ['device_collections']
+            # add any new ones.
-        for attr in collection_attributes:
+            attrs = ['device_collections']
-            attr = attr.strip()
+            if getattr(book, '_new_book', False):
-            for book in self:
+                if prefs['preserve_user_collections']:
+                    # Ensure that the book is in all the book's existing
+                    # collections plus all metadata collections
+                    attrs += collection_attributes
+                else:
+                    # The book's existing collections are ignored. Put the book
+                    # in collections defined by its metadata.
+                    attrs = collection_attributes
+            for attr in attrs:
+                attr = attr.strip()
                 val = getattr(book, attr, None)
                 if not val: continue
                 if isbytestring(val):

src/calibre/devices/usbms/driver.py

@@ -233,6 +233,7 @@ class USBMS(CLI, Device):
             book = self.book_class(prefix, lpath, other=info)
             if book.size is None:
                 book.size = os.stat(self.normalize_path(path)).st_size
+            book._new_book = True # Must be before add_book
             booklists[blist].add_book(book, replace_metadata=True)
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
         debug_print('USBMS: finished adding metadata')
@@ -273,6 +274,9 @@ class USBMS(CLI, Device):
         self.report_progress(1.0, _('Removing books from device metadata listing...'))
         debug_print('USBMS: finished removing metadata for %d books'%(len(paths)))
 
+    # If you override this method and you use book._new_book, then you must
+    # complete the processing before you call this method. The flag is cleared
+    # at the end just before the return
     def sync_booklists(self, booklists, end_session=True):
         debug_print('USBMS: starting sync_booklists')
 
@@ -291,6 +295,12 @@ class USBMS(CLI, Device):
         write_prefix(self._card_a_prefix, 1)
         write_prefix(self._card_b_prefix, 2)
 
+        # Clear the _new_book indication, as we are supposed to be done with
+        # adding books at this point
+        for blist in booklists:
+            for book in blist:
+                book._new_book = False
+
         self.report_progress(1.0, _('Sending metadata to device...'))
         debug_print('USBMS: finished sync_booklists')
 

src/calibre/ebooks/metadata/fetch.py

@@ -273,10 +273,11 @@ def filter_metadata_results(item):
 
 def do_cover_check(item):
     item.has_cover = False
-    try:
+    if item.isbn:
-        item.has_cover = check_for_cover(item.isbn)
+        try:
-    except:
+            item.has_cover = check_for_cover(item.isbn)
-        pass # Cover not found
+        except:
+            pass # Cover not found
 
 def check_for_covers(items):
     threads = [Thread(target=do_cover_check, args=(item,)) for item in items]

src/calibre/ebooks/metadata/isbndb.py

@@ -34,7 +34,8 @@ def fetch_metadata(url, max=100, timeout=5.):
             errmsg = soup.find('errormessage').string
             raise ISBNDBError('Error fetching metadata: '+errmsg)
         total_results = int(book_list['total_results'])
-        np = '&page_number=%s&'%(page_number+1)
+        page_number += 1
+        np = '&page_number=%s&'%page_number
         url = re.sub(r'\&page_number=\d+\&', np, url)
         books.extend(book_list.findAll('bookdata'))
         max -= 1

src/calibre/ebooks/metadata/library_thing.py

@@ -7,6 +7,7 @@ Fetch cover from LibraryThing.com based on ISBN number.
 import sys, socket, os, re
 
 from lxml import html
+import mechanize
 
 from calibre import browser, prints
 from calibre.utils.config import OptionParser
@@ -14,11 +15,17 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
 
+class HeadRequest(mechanize.Request):
+
+    def get_method(self):
+        return 'HEAD'
+
 def check_for_cover(isbn, timeout=5.):
     br = browser()
     br.set_handle_redirect(False)
     try:
-        br.open_novisit(OPENLIBRARY%isbn, timeout=timeout)
+        br.open_novisit(HeadRequest(OPENLIBRARY%isbn), timeout=timeout)
+        return True
     except Exception, e:
         if callable(getattr(e, 'getcode', None)) and e.getcode() == 302:
             return True

src/calibre/ebooks/oeb/base.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
-import os, re, uuid, logging, functools
+import os, re, uuid, logging
 from mimetypes import types_map
 from collections import defaultdict
 from itertools import count
@@ -808,17 +808,17 @@ class Manifest(object):
                         pat = re.compile(r'&(%s);'%('|'.join(user_entities.keys())))
                         data = pat.sub(lambda m:user_entities[m.group(1)], data)
 
-            fromstring = functools.partial(etree.fromstring, parser=RECOVER_PARSER)
+            parser = etree.XMLParser(no_network=True, huge_tree=True)
             # Try with more & more drastic measures to parse
             def first_pass(data):
                 try:
-                    data = fromstring(data)
+                    data = etree.fromstring(data, parser=parser)
                 except etree.XMLSyntaxError, err:
                     self.oeb.log.exception('Initial parse failed:')
                     repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0))
                     data = ENTITY_RE.sub(repl, data)
                     try:
-                        data = fromstring(data)
+                        data = etree.fromstring(data, parser=parser)
                     except etree.XMLSyntaxError, err:
                         self.oeb.logger.warn('Parsing file %r as HTML' % self.href)
                         if err.args and err.args[0].startswith('Excessive depth'):
@@ -832,9 +832,9 @@ class Manifest(object):
                                 elem.text = elem.text.strip('-')
                         data = etree.tostring(data, encoding=unicode)
                         try:
-                            data = fromstring(data)
+                            data = etree.fromstring(data, parser=parser)
                         except etree.XMLSyntaxError:
-                            data = fromstring(data)
+                            data = etree.fromstring(data, parser=RECOVER_PARSER)
                 return data
             data = first_pass(data)
 
@@ -866,12 +866,12 @@ class Manifest(object):
                 data = etree.tostring(data, encoding=unicode)
 
                 try:
-                    data = fromstring(data)
+                    data = etree.fromstring(data, parser=parser)
                 except:
                     data = data.replace(':=', '=').replace(':>', '>')
                     data = data.replace('<http:/>', '')
                     try:
-                        data = fromstring(data)
+                        data = etree.fromstring(data, parser=parser)
                     except etree.XMLSyntaxError:
                         self.oeb.logger.warn('Stripping comments and meta tags from %s'%
                                 self.href)
@@ -882,7 +882,7 @@ class Manifest(object):
                                 "<?xml version='1.0' encoding='utf-8'?><o:p></o:p>",
                                 '')
                         data = data.replace("<?xml version='1.0' encoding='utf-8'??>", '')
-                        data = fromstring(data)
+                        data = etree.fromstring(data)
             elif namespace(data.tag) != XHTML_NS:
                 # OEB_DOC_NS, but possibly others
                 ns = namespace(data.tag)

src/calibre/gui2/__init__.py

@@ -248,7 +248,7 @@ def info_dialog(parent, title, msg, det_msg='', show=False):
 
 class Dispatcher(QObject):
     '''Convenience class to ensure that a function call always happens in the
-    thread the reciver was created in.'''
+    thread the receiver was created in.'''
     dispatch_signal = pyqtSignal(object, object)
 
     def __init__(self, func):
@@ -507,7 +507,7 @@ def pixmap_to_data(pixmap, format='JPEG'):
     buf = QBuffer(ba)
     buf.open(QBuffer.WriteOnly)
     pixmap.save(buf, format)
-    return str(ba.data())
+    return bytes(ba.data())
 
 class ResizableDialog(QDialog):
 

src/calibre/gui2/book_details.py

@@ -173,6 +173,7 @@ class Label(QLabel):
         self.setTextFormat(Qt.RichText)
         self.setText('')
         self.setWordWrap(True)
+        self.setAlignment(Qt.AlignTop)
         self.linkActivated.connect(self.link_activated)
         self._link_clicked = False
         self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
@@ -205,15 +206,15 @@ class BookInfo(QScrollArea):
         rows = render_rows(data)
         rows = u'\n'.join([u'<tr><td valign="top"><b>%s:</b></td><td valign="top">%s</td></tr>'%(k,t) for
             k, t in rows])
+        comments = ''
+        if data.get(_('Comments'), '') not in ('', u'None'):
+            comments = data[_('Comments')]
+            comments = comments_to_html(comments)
         if self.vertical:
-            if _('Comments') in data and data[_('Comments')]:
+            if comments:
-                comments = comments_to_html(data[_('Comments')])
                 rows += u'<tr><td colspan="2">%s</td></tr>'%comments
             self.label.setText(u'<table>%s</table>'%rows)
         else:
-            comments = ''
-            if _('Comments') in data:
-                comments = comments_to_html(data[_('Comments')])
             left_pane = u'<table>%s</table>'%rows
             right_pane = u'<div>%s</div>'%comments
             self.label.setText(u'<table><tr><td valign="top" '

src/calibre/gui2/custom_column_widgets.py

@@ -325,7 +325,7 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
         # Avoid problems with multi-line widgets
         turnover_point = count_non_comment + 1000
     ans = []
-    column = row = 0
+    column = row = comments_row = 0
     for col in cols:
         dt = x[col]['datatype']
         if dt == 'comments':
@@ -337,11 +337,13 @@ def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, pa
             layout.addWidget(w.widgets[c], row, column)
             layout.addWidget(w.widgets[c+1], row, column+1)
             row += 1
+        comments_row = max(comments_row, row)
         if row >= turnover_point:
             column += 2
             turnover_point = count_non_comment + 1000
             row = 0
     if not bulk: # Add the comments fields
+        row = comments_row
         column = 0
         for col in cols:
             dt = x[col]['datatype']

src/calibre/gui2/dialogs/metadata_single.py

@@ -90,7 +90,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     COVER_FETCH_TIMEOUT = 240 # seconds
 
     def do_reset_cover(self, *args):
-        pix = QPixmap(I('book.svg'))
+        pix = QPixmap(I('default_cover.svg'))
         self.cover.setPixmap(pix)
         self.cover_changed = True
         self.cover_data = None
@@ -408,7 +408,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         if cover:
             pm.loadFromData(cover)
         if pm.isNull():
-            pm = QPixmap(I('book.svg'))
+            pm = QPixmap(I('default_cover.svg'))
         else:
             self.cover_data = cover
         self.cover.setPixmap(pm)

src/calibre/gui2/init.py

@@ -56,7 +56,8 @@ class ToolbarMixin(object): # {{{
                 partial(self.edit_metadata, False, bulk=True))
         md.addSeparator()
         md.addAction(_('Download metadata and covers'),
-                partial(self.download_metadata, False, covers=True))
+                partial(self.download_metadata, False, covers=True),
+                Qt.ControlModifier+Qt.Key_D)
         md.addAction(_('Download only metadata'),
                 partial(self.download_metadata, False, covers=False))
         md.addAction(_('Download only covers'),

src/calibre/gui2/library/models.py

@@ -43,6 +43,14 @@ class FormatPath(unicode):
         ans.deleted_after_upload = False
         return ans
 
+_default_image = None
+
+def default_image():
+    global _default_image
+    if _default_image is None:
+        _default_image = QImage(I('default_cover.svg'))
+    return _default_image
+
 class BooksModel(QAbstractTableModel): # {{{
 
     about_to_be_sorted   = pyqtSignal(object, name='aboutToBeSorted')
@@ -71,7 +79,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.book_on_device = None
         self.editable_cols = ['title', 'authors', 'rating', 'publisher',
                               'tags', 'series', 'timestamp', 'pubdate']
-        self.default_image = QImage(I('book.svg'))
+        self.default_image = default_image()
         self.sorted_on = DEFAULT_SORT
         self.sort_history = [self.sorted_on]
         self.last_search = '' # The last search performed on this model

src/calibre/gui2/main.py

@@ -56,6 +56,8 @@ def init_qt(args):
 
 def get_default_library_path():
     fname = _('Calibre Library')
+    if iswindows:
+        fname = 'Calibre Library'
     if isinstance(fname, unicode):
         try:
             fname = fname.encode(filesystem_encoding)

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -579,7 +579,7 @@ void PictureFlowPrivate::resetSlides()
 static QImage prepareSurface(QImage img, int w, int h)
 {
   Qt::TransformationMode mode = Qt::SmoothTransformation;
-  img = img.scaled(w, h, Qt::KeepAspectRatioByExpanding, mode);
+  img = img.scaled(w, h, Qt::IgnoreAspectRatio, mode);
 
   // slightly larger, to accommodate for the reflection
   int hs = int(h * REFLECTION_FACTOR);

src/calibre/library/comments.py

@@ -39,6 +39,11 @@ def comments_to_html(comments):
     if not isinstance(comments, unicode):
         comments = comments.decode(preferred_encoding, 'replace')
 
+    if '<' not in comments:
+        comments = prepare_string_for_xml(comments)
+        comments = comments.replace(u'\n', u'<br />')
+        return u'<p>%s</p>'%comments
+
     # Hackish - ignoring sentences ending or beginning in numbers to avoid
     # confusion with decimal points.
 

src/calibre/library/database2.py

@@ -126,7 +126,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.dbpath = os.path.join(library_path, 'metadata.db')
         self.dbpath = os.environ.get('CALIBRE_OVERRIDE_DATABASE_PATH',
                 self.dbpath)
-        if isinstance(self.dbpath, unicode):
+        if isinstance(self.dbpath, unicode) and not iswindows:
             self.dbpath = self.dbpath.encode(filesystem_encoding)
 
         self.connect()

src/calibre/linux.py

@@ -539,17 +539,20 @@ MIME = '''\
 </mime-info>
 '''
 
-def render_svg(image, dest):
+def render_svg(image, dest, width=128, height=128):
     from PyQt4.QtGui import QPainter, QImage
     from PyQt4.QtSvg import QSvgRenderer
-    svg = QSvgRenderer(image.readAll())
+    image = image.readAll() if hasattr(image, 'readAll') else image
+    svg = QSvgRenderer(image)
     painter = QPainter()
-    image = QImage(128,128,QImage.Format_ARGB32_Premultiplied)
+    image = QImage(width, height, QImage.Format_ARGB32)
     painter.begin(image)
     painter.setRenderHints(QPainter.Antialiasing|QPainter.TextAntialiasing|QPainter.SmoothPixmapTransform|QPainter.HighQualityAntialiasing)
     painter.setCompositionMode(QPainter.CompositionMode_SourceOver)
     svg.render(painter)
     painter.end()
+    if dest is None:
+        return image
     image.save(dest)
 
 def main():

src/calibre/utils/resources.py

@@ -15,19 +15,23 @@ if _dev_path is not None:
     if not os.path.exists(_dev_path):
         _dev_path = None
 
+_path_cache = {}
+
 def get_path(path, data=False):
     global _dev_path
     path = path.replace(os.sep, '/')
-    base = None
+    base = sys.resources_location
     if _dev_path is not None:
+        if path in _path_cache:
+            return _path_cache[path]
         if os.path.exists(os.path.join(_dev_path, *path.split('/'))):
             base = _dev_path
-    if base is None:
+    fpath = os.path.join(base, *path.split('/'))
-        base = sys.resources_location
+    if _dev_path is not None:
-    path = os.path.join(base, *path.split('/'))
+        _path_cache[path] = fpath
     if data:
-        return open(path, 'rb').read()
+        return open(fpath, 'rb').read()
-    return path
+    return fpath
 
 def get_image_path(path, data=False):
     return get_path('images/'+path, data=data)

src/calibre/web/feeds/recipes/model.py

@@ -96,19 +96,21 @@ class NewsItem(NewsTreeItem):
             builtin, custom, scheduler_config, parent):
         NewsTreeItem.__init__(self, builtin, custom, scheduler_config, parent)
         self.urn, self.title = urn, title
+        self.icon = self.default_icon = None
+        self.default_icon = default_icon
         if 'custom:' in self.urn:
             self.icon = custom_icon
-        else:
-            icon = I('news/%s.png'%self.urn[8:])
-            if os.path.exists(icon):
-                self.icon = QVariant(QIcon(icon))
-            else:
-                self.icon = default_icon
 
     def data(self, role):
         if role == Qt.DisplayRole:
             return QVariant(self.title)
         if role == Qt.DecorationRole:
+            if self.icon is None:
+                icon = I('news/%s.png'%self.urn[8:])
+                if os.path.exists(icon):
+                    self.icon = QVariant(QIcon(icon))
+                else:
+                    self.icon = self.default_icon
             return self.icon
         return NONE
 

src/calibre/web/fetch/simple.py

@@ -156,7 +156,7 @@ class RecursiveFetcher(object):
 
         replace = self.prepreprocess_html_ext(soup)
         if replace is not None:
-            soup = BeautifulSoup(xml_to_unicode(src, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
+            soup = BeautifulSoup(xml_to_unicode(replace, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
 
         if self.keep_only_tags:
             body = Tag(soup, 'body')