Sync to trunk.

src/calibre/devices/prs505/driver.py

@@ -4,8 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
 '''
 Device driver for the SONY PRS-505
 '''
-import os
+import os, re, time
-import time
 from itertools import cycle
 
 from calibre import sanitize_file_name as sanitize
@@ -30,12 +29,12 @@ class PRS505(CLI, Device):
 
     VENDOR_NAME        = 'SONY'
     WINDOWS_MAIN_MEM   = 'PRS-505'
-    WINDOWS_CARD_A_MEM = ['PRS-505/UC:MS', 'PRS-505/CE:MS']
+    WINDOWS_CARD_A_MEM = re.compile(r'PRS-505/\S+:MS')
-    WINDOWS_CARD_B_MEM = ['PRS-505/UC:SD', 'PRS-505/CE:SD']
+    WINDOWS_CARD_B_MEM = re.compile(r'PRS-505/\S+:SD')
 
-    OSX_MAIN_MEM = 'Sony PRS-505/UC Media'
+    OSX_MAIN_MEM   = re.compile(r'Sony PRS-505/[^:]+ Media')
-    OSX_CARD_A_MEM = 'Sony PRS-505/UC:MS Media'
+    OSX_CARD_A_MEM = re.compile(r'Sony PRS-505/[^:]+:MS Media')
-    OSX_CARD_B_MEM = 'Sony PRS-505/UC:SD'
+    OSX_CARD_B_MEM = re.compile(r'Sony PRS-505/[^:]+:SD Media')
 
     MAIN_MEMORY_VOLUME_LABEL  = 'Sony Reader Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'Sony Reader Storage Card'
@@ -79,6 +78,7 @@ class PRS505(CLI, Device):
         self.report_progress(1.0, _('Get device information...'))
         return (self.__class__.__name__, '', '', '')
 
+
     def books(self, oncard=None, end_session=True):
         if oncard == 'carda' and not self._card_a_prefix:
             self.report_progress(1.0, _('Getting list of books on device...'))

src/calibre/devices/usbms/device.py

@@ -31,6 +31,8 @@ class Device(DeviceConfig, DevicePlugin):
     WINDOWS_CARD_A_MEM = None
     WINDOWS_CARD_B_MEM = None
 
+    # The following are used by the check_ioreg_line method and can be either:
+    # None, a string, a list of strings or a compiled regular expression
     OSX_MAIN_MEM = None
     OSX_CARD_A_MEM = None
     OSX_CARD_B_MEM = None
@@ -185,6 +187,10 @@ class Device(DeviceConfig, DevicePlugin):
         if device_id is None or \
                 'VEN_' + str(self.VENDOR_NAME).upper() not in pnp_id:
             return False
+
+        if hasattr(device_id, 'search'):
+            return device_id.search(pnp_id) is not None
+
         if isinstance(device_id, basestring):
             device_id = [device_id]
 
@@ -269,6 +275,21 @@ class Device(DeviceConfig, DevicePlugin):
     def osx_sort_names(self, names):
         return names
 
+    def check_ioreg_line(self, line, pat):
+        if pat is None:
+            return False
+        if not line.strip().endswith('<class IOMedia>'):
+            return False
+        if hasattr(pat, 'search'):
+            return pat.search(line) is not None
+        if isinstance(pat, basestring):
+            pat = [pat]
+        for x in pat:
+            if x in line:
+                return True
+        return False
+
+
     def get_osx_mountpoints(self, raw=None):
         raw = self.run_ioreg(raw)
         lines = raw.splitlines()
@@ -285,11 +306,11 @@ class Device(DeviceConfig, DevicePlugin):
                     break
 
         for i, line in enumerate(lines):
-            if self.OSX_MAIN_MEM is not None and line.strip().endswith('<class IOMedia>') and self.OSX_MAIN_MEM in line:
+            if 'main' not in names and self.check_ioreg_line(line, self.OSX_MAIN_MEM):
                 get_dev_node(lines[i+1:], 'main')
-            if self.OSX_CARD_A_MEM is not None and line.strip().endswith('<class IOMedia>') and self.OSX_CARD_A_MEM in line:
+            if 'carda' not in names and self.check_ioreg_line(line, self.OSX_CARD_A_MEM):
                 get_dev_node(lines[i+1:], 'carda')
-            if self.OSX_CARD_B_MEM is not None and line.strip().endswith('<class IOMedia>') and self.OSX_CARD_B_MEM in line:
+            if 'cardb' not in names and self.check_ioreg_line(line, self.OSX_CARD_B_MEM):
                 get_dev_node(lines[i+1:], 'cardb')
             if len(names.keys()) == 3:
                 break

src/calibre/ebooks/epub/input.py

@@ -8,7 +8,7 @@ from itertools import cycle
 
 from lxml import etree
 
-from calibre.customize.conversion import InputFormatPlugin
+from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation
 
 class EPUBInput(InputFormatPlugin):
 
@@ -17,6 +17,8 @@ class EPUBInput(InputFormatPlugin):
     description = 'Convert EPUB files (.epub) to HTML'
     file_types  = set(['epub'])
 
+    recommendations = set([('page_breaks_before', '/', OptionRecommendation.MED)])
+
     @classmethod
     def decrypt_font(cls, key, path):
         raw = open(path, 'rb').read()

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -87,6 +87,7 @@ class MergeMetadata(object):
         if cover_id is not None:
             m.add('cover', cover_id)
 
+
     def set_cover(self, mi, prefer_metadata_cover):
         cdata = ''
         if mi.cover and os.access(mi.cover, os.R_OK):
@@ -94,12 +95,13 @@ class MergeMetadata(object):
         elif mi.cover_data and mi.cover_data[-1]:
             cdata = mi.cover_data[1]
         id = None
-        if 'cover' in self.oeb.guide:
+        old_cover = self.oeb.guide.remove('cover')
-            href = self.oeb.guide['cover'].href
+        self.oeb.guide.remove('titlepage')
-            id = self.oeb.manifest.hrefs[href].id
+        if old_cover is not None:
-            if not prefer_metadata_cover and cdata:
+            if old_cover.href in self.oeb.manifest.hrefs:
-                self.oeb.manifest.hrefs[href]._data = cdata
+                item = self.oeb.manifest.hrefs[old_cover.href]
-        elif cdata:
+                self.oeb.manifest.remove(item)
+        if cdata:
             id, href = self.oeb.manifest.generate('cover', 'cover.jpg')
             self.oeb.manifest.add(id, href, 'image/jpeg', data=cdata)
             self.oeb.guide.add('cover', 'Cover', href)

src/calibre/gui2/convert/metadata.py

@@ -10,7 +10,7 @@ import os, uuid
 
 from PyQt4.Qt import QPixmap, SIGNAL
 
-from calibre.gui2 import choose_images, error_dialog, pixmap_to_data
+from calibre.gui2 import choose_images, error_dialog
 from calibre.gui2.convert.metadata_ui import Ui_Form
 from calibre.ebooks.metadata import authors_to_string, string_to_authors, \
         MetaInformation
@@ -39,8 +39,6 @@ class MetadataWidget(Widget, Ui_Form):
 
         mi = self.db.get_metadata(self.book_id, index_is_id=True)
         self.title.setText(mi.title)
-#        if mi.authors:
-#            self.author.setCurrentIndex(self.author.findText(authors_to_string(mi.authors)))
         if mi.publisher:
             self.publisher.setCurrentIndex(self.publisher.findText(mi.publisher))
         self.author_sort.setText(mi.author_sort if mi.author_sort else '')
@@ -187,7 +185,7 @@ class MetadataWidget(Widget, Ui_Form):
             opf.render(self.opf_file)
             self.opf_file.close()
             if self.cover_changed:
-                self.db.set_cover(self.book_id, pixmap_to_data(self.cover.pixmap()))
+                self.db.set_cover(self.book_id, self.cover.pixmap())
             cover = self.db.cover(self.book_id, index_is_id=True)
             if cover:
                 cf = PersistentTemporaryFile('.jpeg')

src/calibre/gui2/dialogs/metadata_single.py

@@ -16,7 +16,7 @@ from PyQt4.QtCore import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread,
 from PyQt4.QtGui import QPixmap, QListWidgetItem, QErrorMessage, QDialog
 
 from calibre.gui2 import qstring_to_unicode, error_dialog, file_icon_provider, \
-                           choose_files, pixmap_to_data, choose_images, ResizableDialog
+                           choose_files, choose_images, ResizableDialog
 from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog
 from calibre.gui2.dialogs.fetch_metadata import FetchMetadata
 from calibre.gui2.dialogs.tag_editor import TagEditor
@@ -512,7 +512,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.db.set_pubdate(self.id, datetime(d.year(), d.month(), d.day()))
 
         if self.cover_changed:
-            self.db.set_cover(self.id, pixmap_to_data(self.cover.pixmap()))
+            self.db.set_cover(self.id, self.cover.pixmap())
         QDialog.accept(self)
         if callable(self.accepted_callback):
             self.accepted_callback(self.id)

src/calibre/library/database2.py

@@ -11,9 +11,15 @@ import os, re, sys, shutil, cStringIO, glob, collections, textwrap, \
 from itertools import repeat
 from datetime import datetime
 
-from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock
+from PyQt4.QtCore import QThread, QReadWriteLock
-from PyQt4.QtGui import QApplication, QImage
+try:
-__app = None
+    from PIL import Image as PILImage
+    PILImage
+except ImportError:
+    import Image as PILImage
+
+
+from PyQt4.QtGui import QImage
 
 from calibre.ebooks.metadata import title_sort
 from calibre.library.database import LibraryDatabase
@@ -40,11 +46,15 @@ def delete_file(path):
     except:
         os.remove(path)
 
-def delete_tree(path):
+def delete_tree(path, permanent=False):
-    try:
+    if permanent:
-        winshell.delete_file(path, silent=True, no_confirm=True)
-    except:
         shutil.rmtree(path)
+    else:
+        try:
+            if not permanent:
+                winshell.delete_file(path, silent=True, no_confirm=True)
+        except:
+            shutil.rmtree(path)
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -661,9 +671,9 @@ class LibraryDatabase2(LibraryDatabase):
             name = name[:-1]
         return name
 
-    def rmtree(self, path):
+    def rmtree(self, path, permanent=False):
         if not self.normpath(self.library_path).startswith(self.normpath(path)):
-            delete_tree(path)
+            delete_tree(path, permanent=permanent)
 
     def normpath(self, path):
         path = os.path.abspath(os.path.realpath(path))
@@ -715,10 +725,10 @@ class LibraryDatabase2(LibraryDatabase):
         # Delete not needed directories
         if current_path and os.path.exists(spath):
             if self.normpath(spath) != self.normpath(tpath):
-                self.rmtree(spath)
+                self.rmtree(spath, permanent=True)
                 parent  = os.path.dirname(spath)
                 if len(os.listdir(parent)) == 0:
-                    self.rmtree(parent)
+                    self.rmtree(parent, permanent=True)
 
     def add_listener(self, listener):
         '''
@@ -815,14 +825,11 @@ class LibraryDatabase2(LibraryDatabase):
         if callable(getattr(data, 'save', None)):
             data.save(path)
         else:
-            if not QCoreApplication.instance():
+            f = data
-                global __app
+            if not callable(getattr(data, 'read', None)):
-                __app = QApplication([])
+                f = cStringIO.StringIO(data)
-            p = QImage()
+            im = PILImage.open(f)
-            if callable(getattr(data, 'read', None)):
+            im.convert('RGB').save(path, 'JPEG')
-                data = data.read()
-            p.loadFromData(data)
-            p.save(path)
 
     def all_formats(self):
         formats = self.conn.get('SELECT format from data')
@@ -1524,6 +1531,7 @@ class LibraryDatabase2(LibraryDatabase):
         return data
 
     def migrate_old(self, db, progress):
+        from PyQt4.QtCore import QCoreApplication
         header = _(u'<p>Migrating old database to ebook library in %s<br><center>')%self.library_path
         progress.setValue(0)
         progress.setLabelText(header)

src/calibre/web/feeds/recipes/recipe_usatoday.py

@@ -14,14 +14,26 @@ class USAToday(BasicNewsRecipe):
     title = 'USA Today'
     timefmt  = ' [%d %b %Y]'
     max_articles_per_feed = 20
-    no_stylesheets = True
     language = _('English')
+    __author__ = _('Kovid Goyal and Sujata Raman')
+
+    no_stylesheets = True
     extra_css = '''
-    .inside-head { font: x-large bold } 
+            .inside-head{font-family:Arial,Helvetica,sans-serif; font-size:large; font-weight:bold }
-    .inside-head2 { font: x-large bold }
+            .inside-head2{font-family:Arial,Helvetica,sans-serif; font-size:large; font-weight:bold }
-    .inside-head3 { font: x-large bold }
+            .inside-head3{font-family:Arial,Helvetica,sans-serif; font-size:large; font-weight:bold }
-    .byLine { font: large }
+            h3{font-family:Arial,Helvetica,sans-serif; font-size:large; font-weight:bold }
-    '''
+            h4{font-family:Arial,Helvetica,sans-serif; font-size:x-small; font-weight:bold }
+            .side-by-side{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+            #byLineTag{font-family:Arial,Helvetica,sans-serif; font-size:xx-small;}
+            .inside-copy{font-family:Arial,Helvetica,sans-serif; font-size:x-small;text-align:left}
+            .caption{font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
+            '''
+    remove_tags = [
+                    dict(name='div', attrs={'class':'inside-copy'}),
+                    {'class':['tagListLabel','piped-taglist-string',]}
+                  ]
+
     html2lrf_options = ['--ignore-tables']
 
     preprocess_regexps = [
@@ -29,6 +41,7 @@ class USAToday(BasicNewsRecipe):
         (re.compile(r'<!--Article End-->.*?</BODY>', re.IGNORECASE | re.DOTALL), lambda match : '</BODY>'),
         ]
 
+
     feeds =  [
                 ('Top Headlines', 'http://rssfeeds.usatoday.com/usatoday-NewsTopStories'),
                 ('Sport Headlines', 'http://rssfeeds.usatoday.com/UsatodaycomSports-TopStories'),