Pull from trunck

Changelog.yaml

@@ -4,6 +4,38 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.1
+  date: 2010-06-04
+
+  new features:
+    - title: "Content server: Add option to control category groupiong in OPDS feeds"
+
+    - title: "Make the book details pane occupy the full lower part of the window"
+
+    - title: "Add true and false searches for date based columns"
+      tickets: [5717]
+
+  bug fixes:
+    - title: "iPad driver: Various bug fixes."
+
+    - title: "SONY driver: Fix Launcher partition being detected as storage card in linux"
+
+    - title: "Fix news downloading breaking on windows systems with local encoding other than UTF-8."
+
+    - title: "SONY driver: Fix problem caused by null titles"
+
+    - title: "Make the new splash screen not always stay on top"
+      tickets: [5700]
+
+    - title: "When setting an image with transparent pixels as the book cover, overlay it on a white background first. Fixes transparent covers getting random backgrounds."
+
+    - title: "Content server: Fix stanza integration when entering the server URL my hand"
+
+  improved recipes:
+    - Gizmodo
+    - Vreme
+
+
 - version: 0.7.0
   date: 2010-06-04
 

resources/recipes/cbc_canada.recipe

@@ -0,0 +1,25 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1275798572(BasicNewsRecipe):
+    title          = u'CBC Canada'
+    publisher             = 'www.cbc.ca'
+    language = 'en_CA'
+    __author__ = 'rty'
+    category              = 'news'
+    oldest_article = 4
+    max_articles_per_feed = 100
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'en'
+    masthead_url = 'http://www.cbc.ca/includes/gfx/cbcnews_logo_09.gif'
+    cover_url = 'http://img692.imageshack.us/img692/2814/cbc.png'
+    keep_only_tags = [dict(name='div', attrs={'id':['storyhead','storybody']})]
+    remove_tags_after  = dict(id=['socialtools'])
+    feeds          = [(u'Top Stories', u'http://rss.cbc.ca/lineup/topstories.xml'),
+                          (u'World', u'http://rss.cbc.ca/lineup/world.xml'),
+                          (u'National', u'http://rss.cbc.ca/lineup/canada.xml'),
+                          (u'Manitoba', u'http://rss.cbc.ca/lineup/canada-manitoba.xml'),
+                          (u'Politics', u'http://rss.cbc.ca/lineup/politics.xml'),
+                          (u'Tech & Science', u'http://rss.cbc.ca/lineup/technology.xml'),
+                          (u'Books', u'http://rss.cbc.ca/lineup/arts-books.xml')]

resources/recipes/gizmodo.recipe

@@ -17,7 +17,7 @@ class Gizmodo(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     encoding              = 'utf-8'
-    use_embedded_content  = True
+    use_embedded_content  = False
     language              = 'en'
     masthead_url          = 'http://cache.gawkerassets.com/assets/gizmodo.com/img/logo.png'
     extra_css             = ' body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif} img{margin-bottom: 1em} '
@@ -29,9 +29,11 @@ class Gizmodo(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes = ['width','height']
-    remove_tags       = [dict(name='div',attrs={'class':'feedflare'})]
-    remove_tags_after = dict(name='div',attrs={'class':'feedflare'})
+    remove_attributes  = ['width','height']
+    keep_only_tags     = [dict(attrs={'class':'content permalink'})]
+    remove_tags_before = dict(name='h1')
+    remove_tags        = [dict(attrs={'class':'contactinfo'})]
+    remove_tags_after  = dict(attrs={'class':'contactinfo'})
 
     feeds = [(u'Articles', u'http://feeds.gawker.com/gizmodo/full')]
 

resources/recipes/haaretz_en.recipe

@@ -0,0 +1,57 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+haaretz.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Haaretz_en(BasicNewsRecipe):
+    title                 = 'Haaretz in English'
+    __author__            = 'Darko Miletic'
+    description           = 'Haaretz.com, the online edition of Haaretz Newspaper in Israel, and analysis from Israel and the Middle East. Haaretz.com provides extensive and in-depth coverage of Israel, the Jewish World and the Middle East, including defense, diplomacy, the Arab-Israeli conflict, the peace process, Israeli politics, Jerusalem affairs, international relations, Iran, Iraq, Syria, Lebanon, the Palestinian Authority, the West Bank and the Gaza Strip, the Israeli business world and Jewish life in Israel and the Diaspora. '
+    publisher             = 'haaretz.com'
+    category              = 'news, politics, Israel'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'cp1252'
+    use_embedded_content  = False
+    language              = 'en_IL'
+    publication_type      = 'newspaper'
+    remove_empty_feeds    = True
+    masthead_url          = 'http://www.haaretz.com/images/logos/logoGrey.gif'
+    extra_css             = ' body{font-family: Verdana,Arial,Helvetica,sans-serif } '
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [dict(name='div', attrs={'class':['rightcol']}),dict(name='table')]
+    remove_tags_before = dict(name='h1')
+    remove_tags_after  = dict(attrs={'id':'innerArticle'})
+    keep_only_tags     = [dict(attrs={'id':'content'})]
+
+
+    feeds = [
+              (u'Opinion'               , u'http://www.haaretz.com/cmlink/opinion-rss-1.209234?localLinksEnabled=false'   )
+             ,(u'Defense and diplomacy' , u'http://www.haaretz.com/cmlink/defense-and-diplomacy-rss-1.208894?localLinksEnabled=false')
+             ,(u'National'              , u'http://www.haaretz.com/cmlink/national-rss-1.208896?localLinksEnabled=false'       )
+             ,(u'International'         , u'http://www.haaretz.com/cmlink/international-rss-1.208898?localLinksEnabled=false'      )
+             ,(u'Jewish World'          , u'http://www.haaretz.com/cmlink/jewish-world-rss-1.209085?localLinksEnabled=false'        )
+             ,(u'Business'              , u'http://www.haaretz.com/cmlink/business-print-rss-1.264904?localLinksEnabled=false'     )
+             ,(u'Real Estate'           , u'http://www.haaretz.com/cmlink/real-estate-print-rss-1.264977?localLinksEnabled=false'      )
+             ,(u'Features'              , u'http://www.haaretz.com/cmlink/features-print-rss-1.264912?localLinksEnabled=false'          )
+             ,(u'Arts and leisure'      , u'http://www.haaretz.com/cmlink/arts-and-leisure-rss-1.286090?localLinksEnabled=false'       )
+             ,(u'Books'                 , u'http://www.haaretz.com/cmlink/books-rss-1.264947?localLinksEnabled=false'         )
+             ,(u'Food and Wine'         , u'http://www.haaretz.com/cmlink/food-and-wine-print-rss-1.265034?localLinksEnabled=false'      )
+             ,(u'Sports'                , u'http://www.haaretz.com/cmlink/sports-rss-1.286092?localLinksEnabled=false'          )
+            ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

resources/recipes/nytimes.recipe

@@ -16,7 +16,7 @@ class NYTimes(BasicNewsRecipe):
 
     title       = 'New York Times Top Stories'
     __author__  = 'GRiker'
-    language = _('English')
+    language = 'en'
     description = 'Top Stories from the New York Times'
 
     # List of sections typically included in Top Stories.  Use a keyword from the

resources/recipes/ourdailybread.recipe

@@ -1,9 +1,7 @@
-#!/usr/bin/env  python
-
 __license__   = 'GPL v3'
-__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
-rbc.org
+odb.org
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
@@ -11,27 +9,29 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class OurDailyBread(BasicNewsRecipe):
     title                 = 'Our Daily Bread'
     __author__            = 'Darko Miletic and Sujata Raman'
-    description           = 'Religion'
+    description           = "Our Daily Bread is a daily devotional from RBC Ministries which helps readers spend time each day in God's Word."
     oldest_article        = 15
-    language = 'en'
-    lang = 'en'
-
+    language              = 'en'
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
-    category              = 'religion'
+    category              = 'ODB, Daily Devotional, Bible, Christian Devotional, Devotional, RBC Ministries, Our Daily Bread, Devotionals, Daily Devotionals, Christian Devotionals, Faith, Bible Study, Bible Studies, Scripture, RBC, religion'
     encoding              = 'utf-8'
-  
+
     conversion_options = {
-                             'comments'    : description
-                            ,'tags'        : category
-                            ,'language'    : 'en'
+                             'comments'         : description
+                            ,'tags'             : category
+                            ,'language'         : language
+                            ,'linearize_tables' : True
                          }
 
-    keep_only_tags = [dict(name='div', attrs={'class':['altbg','text']})]
+    keep_only_tags    = [dict(attrs={'class':'module-content'})]
+    remove_tags       = [
+                           dict(attrs={'id':'article-zoom'})
+                          ,dict(attrs={'class':'listen-now-box'})
+                        ]
+    remove_tags_after = dict(attrs={'class':'readable-area'})
 
-    remove_tags = [dict(name='div', attrs={'id':['ctl00_cphPrimary_pnlBookCover']}),
-                   ]
     extra_css = '''
                 .text{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}
                 .devotionalTitle{font-family:Arial,Helvetica,sans-serif; font-size:large; font-weight: bold;}
@@ -40,14 +40,9 @@ class OurDailyBread(BasicNewsRecipe):
                 a{color:#000000;font-family:Arial,Helvetica,sans-serif; font-size:x-small;}
                 '''
 
-    feeds          = [(u'Our Daily Bread', u'http://www.rbc.org/rss.ashx?id=50398')]
+    feeds          = [(u'Our Daily Bread', u'http://odb.org/feed/')]
 
     def preprocess_html(self, soup):
-        soup.html['xml:lang'] = self.lang
-        soup.html['lang']     = self.lang
-        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=' + self.encoding + '">'
-        soup.head.insert(0,mtag)
-
         return self.adeify_images(soup)
 
     def get_cover_url(self):
@@ -61,3 +56,4 @@ class OurDailyBread(BasicNewsRecipe):
            cover_url = a.img['src']
 
         return cover_url
+

resources/recipes/sueddeutsche.recipe

@@ -19,10 +19,10 @@ class Sueddeutsche(BasicNewsRecipe):
     no_stylesheets = True
     language = 'de'
 
-    encoding = 'iso-8859-15'
+    encoding = 'utf-8'
     remove_javascript = True
 
-    
+
     remove_tags = [ dict(name='link'), dict(name='iframe'),
                     dict(name='div', attrs={'id':["bookmarking","themenbox","artikelfoot","CAD_AD",
 						  "SKY_AD","NT1_AD","navbar1","sdesiteheader"]}),

resources/recipes/vreme.recipe

@@ -52,10 +52,12 @@ class Vreme(BasicNewsRecipe):
     def parse_index(self):
         articles = []
         soup = self.index_to_soup(self.INDEX)
-
+        cover_item = soup.find('div',attrs={'id':'najava'})
+        if cover_item:
+           self.cover_url = self.INDEX + cover_item.img['src']      
         for item in soup.findAll(['h3','h4']):
-            description = ''
-            title_prefix = ''
+            description = u''
+            title_prefix = u''
             feed_link = item.find('a')
             if feed_link and feed_link.has_key('href') and feed_link['href'].startswith('/cms/view.php'):
                 url   = self.INDEX + feed_link['href']
@@ -67,7 +69,7 @@ class Vreme(BasicNewsRecipe):
                                  ,'url'        :url
                                  ,'description':description
                                 })
-        return [(soup.head.title.string, articles)]
+        return [('Nedeljnik Vreme', articles)]
 
     remove_tags = [
                     dict(name=['object','link'])
@@ -76,11 +78,3 @@ class Vreme(BasicNewsRecipe):
 
     def print_version(self, url):
         return url + '&print=yes'
-
-    def get_cover_url(self):
-        cover_url = None
-        soup = self.index_to_soup(self.INDEX)
-        cover_item = soup.find('div',attrs={'id':'najava'})
-        if cover_item:
-           cover_url = self.INDEX + cover_item.img['src']
-        return cover_url

src/calibre/__init__.py

@@ -41,6 +41,8 @@ mimetypes.add_type('application/vnd.palm',                '.pdb')
 mimetypes.add_type('application/x-mobipocket-ebook',      '.mobi')
 mimetypes.add_type('application/x-mobipocket-ebook',      '.prc')
 mimetypes.add_type('application/x-mobipocket-ebook',      '.azw')
+mimetypes.add_type('application/x-cbz',                   '.cbz')
+mimetypes.add_type('application/x-cbr',                   '.cbr')
 mimetypes.add_type('application/x-koboreader-ebook',      '.kobo') 
 mimetypes.add_type('image/wmf',                           '.wmf')
 guess_type = mimetypes.guess_type

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.7.0'
+__version__   = '0.7.1'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/devices/apple/driver.py

@@ -5,15 +5,15 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, os, re, shutil, sys, tempfile, time, zipfile
+import cStringIO, os, re, shutil, subprocess, sys, tempfile, time, zipfile
 
 from calibre.constants import DEBUG
 from calibre import fit_image
 from calibre.constants import isosx, iswindows
 from calibre.devices.interface import DevicePlugin
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.library.server.utils import strftime
-from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import Config, config_dir
 from calibre.utils.date import parse_date
 from calibre.utils.logging import Log
@@ -22,10 +22,46 @@ from calibre.devices.errors import UserFeedback
 from PIL import Image as PILImage
 
 if isosx:
-    import appscript
+    try:
+        import appscript
+        appscript
+    except:
+        # appscript fails to load on 10.4
+        appscript = None
 
 if iswindows:
     import pythoncom, win32com.client
+
+class ITUNES(DevicePlugin):
+    '''
+            try:
+                pythoncom.CoInitialize()
+            finally:
+                pythoncom.CoUninitialize()
+    '''
+
+    name = 'Apple device interface'
+    gui_name = 'Apple device'
+    icon = I('devices/ipad.png')
+    description    = _('Communicate with iBooks through iTunes.')
+    supported_platforms = ['osx','windows']
+    author = 'GRiker'
+    #: The version of this plugin as a 3-tuple (major, minor, revision)
+    version        = (0, 4, 0)
+
+    OPEN_FEEDBACK_MESSAGE = _(
+        'Apple device detected, launching iTunes, please wait ...')
+
+    FORMATS = ['epub']
+
+    # Product IDs:
+    #  0x1292:iPhone 3G
+    #  0x129a:iPad
+    VENDOR_ID = [0x05ac]
+    PRODUCT_ID = [0x129a]
+    BCD = [0x01]
+
+    # iTunes enumerations
     Sources = [
                 'Unknown',
                 'Library',
@@ -43,40 +79,34 @@ if iswindows:
                 'BMP'
                 ]
 
-class ITUNES(DevicePlugin):
-    '''
-            try:
-                pythoncom.CoInitialize()
-            finally:
-                pythoncom.CoUninitialize()
-    '''
+    PlaylistKind = [
+                'Unknown',
+                'Library',
+                'User',
+                'CD',
+                'Device',
+                'Radio Tuner'
+                ]
 
-    name = 'Apple device interface'
-    gui_name = 'Apple device'
-    icon = I('devices/ipad.png')
-    description    = _('Communicate with iBooks through iTunes.')
-    supported_platforms = ['osx','windows']
-    author = 'GRiker'
-    #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (1, 0, 0)
-
-    OPEN_FEEDBACK_MESSAGE = _(
-        'Apple device detected, launching iTunes, please wait ...')
-
-    FORMATS = ['epub']
-
-    # Product IDs:
-    #  0x1292:iPhone 3G
-    #  0x129a:iPad
-    VENDOR_ID = [0x05ac]
-    PRODUCT_ID = [0x129a]
-    BCD = [0x01]
+    PlaylistSpecialKind = [
+                'Unknown',
+                'Purchased Music',
+                'Party Shuffle',
+                'Podcasts',
+                'Folder',
+                'Video',
+                'Music',
+                'Movies',
+                'TV Shows',
+                'Books',
+                ]
 
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
     ejected = False
     iTunes= None
+    iTunes_media = None
     log = Log()
     path_template = 'iTunes/%s - %s.epub'
     problem_titles = []
@@ -109,12 +139,12 @@ class ITUNES(DevicePlugin):
                     self.log.info( "ITUNES.add_books_to_metadata()")
                     self._dump_update_list('add_books_to_metadata()')
                 for (j,p_book) in enumerate(self.update_list):
-                    self.log.info("ITUNES.add_books_to_metadata(): looking for %s" %
+                    self.log.info("ITUNES.add_books_to_metadata():\n looking for %s" %
                         str(p_book['lib_book'])[-9:])
                     for i,bl_book in enumerate(booklists[0]):
                         if bl_book.library_id == p_book['lib_book']:
                             booklists[0].pop(i)
-                            self.log.info("ITUNES.add_books_to_metadata(): removing %s %s" %
+                            self.log.info("ITUNES.add_books_to_metadata():\n removing %s %s" %
                                 (p_book['title'], str(p_book['lib_book'])[-9:]))
                             break
                     else:
@@ -151,8 +181,9 @@ class ITUNES(DevicePlugin):
                 self.log.info(" adding '%s' by '%s' to booklists[0]" %
                     (new_book.title, new_book.author))
             booklists[0].append(new_book)
-        if DEBUG:
-            self._dump_booklist(booklists[0],'after add_books_to_metadata()')
+
+#         if DEBUG:
+#             self._dump_booklist(booklists[0],'after add_books_to_metadata()')
 
     def books(self, oncard=None, end_session=True):
         """
@@ -268,6 +299,8 @@ class ITUNES(DevicePlugin):
         instantiate iTunes if necessary
         This gets called ~1x/second while device fingerprint is sensed
         '''
+        if appscript is None:
+            return False
 
         if self.iTunes:
             # Check for connected book-capable device
@@ -452,12 +485,14 @@ class ITUNES(DevicePlugin):
         if isosx:
             self.iTunes.eject(self.sources['iPod'])
         elif iswindows:
-            try:
-                pythoncom.CoInitialize()
-                self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                self.iTunes.sources.ItemByName(self.sources['iPod']).EjectIPod()
-            finally:
-                pythoncom.CoUninitialize()
+            if 'iPod' in self.sources:
+                try:
+                    pythoncom.CoInitialize()
+                    self.iTunes = win32com.client.Dispatch("iTunes.Application")
+                    self.iTunes.sources.ItemByName(self.sources['iPod']).EjectIPod()
+
+                finally:
+                    pythoncom.CoUninitialize()
 
         self.iTunes = None
         self.sources = None
@@ -562,7 +597,7 @@ class ITUNES(DevicePlugin):
                                 L{books}(oncard='cardb')).
         '''
         if DEBUG:
-            self.log.info("ITUNES.remove_books_from_metadata():")
+            self.log.info("ITUNES.remove_books_from_metadata()")
         for path in paths:
             if self.cached_books[path]['lib_book']:
                 # Remove from the booklist
@@ -572,15 +607,15 @@ class ITUNES(DevicePlugin):
                         booklists[0].pop(i)
                         break
                 else:
-                    self.log.error("ITUNES.remove_books_from_metadata(): '%s' not found in self.cached_book" % path)
+                    self.log.error(" '%s' not found in self.cached_book" % path)
 
                 # Remove from cached_books
                 self.cached_books.pop(path)
                 if DEBUG:
-                    self.log.info("ITUNES.remove_books_from_metadata(): Removing '%s' from self.cached_books" % path)
-                    self._dump_cached_books('remove_books_from_metadata()')
+                    self.log.info(" Removing '%s' from self.cached_books" % path)
+#                    self._dump_cached_books('remove_books_from_metadata()')
             else:
-                self.log.warning("ITUNES.remove_books_from_metadata(): skipping purchased book, can't remove via automation interface")
+                self.log.warning(" skipping purchased book, can't remove via automation interface")
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -624,54 +659,13 @@ class ITUNES(DevicePlugin):
                                 L{books}(oncard='cardb')).
         '''
         if DEBUG:
-            self.log.info("ITUNES:sync_booklists():")
+            self.log.info("ITUNES:sync_booklists()")
         if self.update_needed:
             if DEBUG:
                 self.log.info(' calling _update_device')
-            self._update_device(msg=self.update_msg)
-            self.update_needed = False
-
-        # Get actual size of updated books on device
-        if self.update_list:
-            if DEBUG:
-                self._dump_update_list(header='sync_booklists()')
-            if isosx:
-                for updated_book in self.update_list:
-                    size_on_device = self._get_device_book_size(updated_book['title'],
-                                                                updated_book['author'][0])
-                    if size_on_device:
-                        for book in booklists[0]:
-                            if book.title == updated_book['title'] and \
-                               book.author == updated_book['author']:
-                                break
-                        else:
-                            self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
-
-                    else:
-                        self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
-
-            elif iswindows:
-                try:
-                    pythoncom.CoInitialize()
-                    self.iTunes = win32com.client.Dispatch("iTunes.Application")
-
-                    for updated_book in self.update_list:
-                        size_on_device = self._get_device_book_size(updated_book['title'], updated_book['author'])
-                        if size_on_device:
-                            for book in booklists[0]:
-                                if book.title == updated_book['title'] and \
-                                   book.author[0] == updated_book['author']:
-                                    book.size = size_on_device
-                                    break
-                            else:
-                                self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
-
-                        else:
-                            self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
-                finally:
-                    pythoncom.CoUninitialize()
-
+            self._update_device(msg=self.update_msg, wait=False)
             self.update_list = []
+            self.update_needed = False
 
         # Inform user of any problem books
         if self.problem_titles:
@@ -729,22 +723,50 @@ class ITUNES(DevicePlugin):
         self.problem_msg = _("Some cover art could not be converted.\n"
                                      "Click 'Show Details' for a list.")
 
+        if DEBUG:
+            self.log.info("ITUNES.upload_books()")
+            self._dump_files(files, header='upload_books()')
+#            self._dump_cached_books('upload_books()')
+            self._dump_update_list('upload_books()')
+
         if isosx:
-            if DEBUG:
-                self.log.info("ITUNES.upload_books():")
-                self._dump_files(files, header='upload_books()')
-                self._dump_cached_books('upload_books()')
-                self._dump_update_list('upload_books()')
             for (i,file) in enumerate(files):
+                # Delete existing from Library|Books
+                # Add to self.update_list for deletion from booklist[0] during add_books_to_metadata
+
+                '''
+                # ---------------------------
+                # PROVISIONAL
+                # Use the cover to find the database storage point of the epub
+                # Pass database copy to iTunes instead of the temporary file
+
+                if False:
+                    if DEBUG:
+                        self.log.info(" processing '%s'" % metadata[i].title)
+                        self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
+                        self.log.info(" cover: %s" % metadata[i].cover)
+
+                    calibre_database_item = False
+                    if metadata[i].cover:
+                        passed_file = file
+                        storage_path =  os.path.split(metadata[i].cover)[0]
+                        try:
+                            database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
+                            file = os.path.join(storage_path,database_epub)
+                            calibre_database_item = True
+                            self.log.info(" using database file: %s" % file)
+                        except:
+                            self.log.info(" could not find epub in %s" % storage_path)
+                    else:
+                        self.log.info(" no cover available, using temp file")
+                # ---------------------------
+                '''
+
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
-                # Delete existing from Library|Books, add to self.update_list
-                # for deletion from booklist[0] during add_books_to_metadata
                 if path in self.cached_books:
                     if DEBUG:
                         self.log.info(" adding '%s' by %s to self.update_list" %
                                       (self.cached_books[path]['title'],self.cached_books[path]['author']))
-
-                    # *** Second time a book is updated the author is a list ***
                     self.update_list.append(self.cached_books[path])
 
                     if DEBUG:
@@ -752,10 +774,12 @@ class ITUNES(DevicePlugin):
                     self._remove_from_iTunes(self.cached_books[path])
 
                 # Add to iTunes Library|Books
-                if isinstance(file,PersistentTemporaryFile):
-                    added = self.iTunes.add(appscript.mactypes.File(file._name))
-                else:
-                    added = self.iTunes.add(appscript.mactypes.File(file))
+                fpath = file
+                if getattr(file, 'orig_file_path', None) is not None:
+                    fpath = file.orig_file_path
+                elif getattr(file, 'name', None) is not None:
+                    fpath = file.name
+                added = self.iTunes.add(appscript.mactypes.File(fpath))
 
                 thumb = None
                 if metadata[i].cover:
@@ -792,16 +816,17 @@ class ITUNES(DevicePlugin):
                 this_book.device_collections = []
                 this_book.library_id = added
                 this_book.path = path
-                this_book.size = added.size()   # Updated later from actual storage size
+                this_book.size = self._get_device_book_size(fpath, added.size())
                 this_book.thumbnail = thumb
                 this_book.iTunes_id = added
 
                 new_booklist.append(this_book)
 
-                # Flesh out the iTunes metadata
-                added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
+                # Populate the iTunes metadata
                 if metadata[i].comments:
                     added.comment.set(strip_tags.sub('',metadata[i].comments))
+                added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
+                added.enabled.set(True)
                 if metadata[i].rating:
                     added.rating.set(metadata[i].rating*10)
                 added.sort_artist.set(metadata[i].author_sort.title())
@@ -825,20 +850,67 @@ class ITUNES(DevicePlugin):
                 # Report progress
                 if self.report_progress is not None:
                     self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))
+
         elif iswindows:
             try:
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                lib = self.iTunes.sources.ItemByName('Library')
-                lib_playlists = [pl.Name for pl in lib.Playlists]
-                if not 'Books' in lib_playlists:
-                    self.log.error(" no 'Books' playlist in Library")
-                library_books = lib.Playlists.ItemByName('Books')
+
+                for source in self.iTunes.sources:
+                    if source.Kind == self.Sources.index('Library'):
+                        lib = source
+                        if DEBUG:
+                            self.log.info(" Library source: '%s'  kind: %s" % (lib.Name, self.Sources[lib.Kind]))
+                        break
+                else:
+                    if DEBUG:
+                        self.log.info(" Library source not found")
+
+                if lib is not None:
+                    lib_books = None
+                    for pl in lib.Playlists:
+                        if self.PlaylistKind[pl.Kind] == 'User' and self.PlaylistSpecialKind[pl.SpecialKind] == 'Books':
+                            if DEBUG:
+                                self.log.info(" Books playlist: '%s' special_kind: '%s'" % (pl.Name, self.PlaylistSpecialKind[pl.SpecialKind]))
+                            lib_books = pl
+                            break
+                    else:
+                        if DEBUG:
+                            self.log.error(" no Books playlist found")
 
                 for (i,file) in enumerate(files):
-                    path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     # Delete existing from Library|Books, add to self.update_list
                     # for deletion from booklist[0] during add_books_to_metadata
+
+                    '''
+                    # ---------------------------
+                    # PROVISIONAL
+                    # Use the cover to find the database storage point of the epub
+                    # Pass database copy to iTunes instead of the temporary file
+
+                    if False:
+                        if DEBUG:
+                            self.log.info(" processing '%s'" % metadata[i].title)
+                            self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
+                            self.log.info(" cover: %s" % metadata[i].cover)
+
+                        calibre_database_item = False
+                        if metadata[i].cover:
+                            passed_file = file
+                            storage_path =  os.path.split(metadata[i].cover)[0]
+                            try:
+                                database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
+                                file = os.path.join(storage_path,database_epub)
+                                calibre_database_item = True
+                                self.log.info(" using database file: %s" % file)
+                            except:
+                                self.log.info(" could not find epub in %s" % storage_path)
+                        else:
+                            self.log.info(" no cover available, using temp file")
+                    # ---------------------------
+                    '''
+
+                    path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     if path in self.cached_books:
                         self.update_list.append(self.cached_books[path])
 
@@ -851,12 +923,15 @@ class ITUNES(DevicePlugin):
                             self.log.info(" '%s' not in cached_books" % metadata[i].title)
 
                     # Add to iTunes Library|Books
-                    if isinstance(file,PersistentTemporaryFile):
-                        op_status = library_books.AddFile(file._name)
-                        self.log.info("ITUNES.upload_books():\n iTunes adding '%s'" % file._name)
-                    else:
-                        op_status = library_books.AddFile(file)
-                        self.log.info(" iTunes adding '%s'" % file)
+                    fpath = file
+                    if getattr(file, 'orig_file_path', None) is not None:
+                        fpath = file.orig_file_path
+                    elif getattr(file, 'name', None) is not None:
+                        fpath = file.name
+
+                    op_status = lib_books.AddFile(fpath)
+                    self.log.info("ITUNES.upload_books():\n iTunes adding '%s'"
+                            % fpath)
 
                     if DEBUG:
                         sys.stdout.write(" iTunes copying '%s' ..." % metadata[i].title)
@@ -936,9 +1011,10 @@ class ITUNES(DevicePlugin):
                     new_booklist.append(this_book)
 
                     # Flesh out the iTunes metadata
-                    added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
                     if metadata[i].comments:
                         added.Comment = (strip_tags.sub('',metadata[i].comments))
+                    added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
+                    added.Enabled = True
                     if metadata[i].rating:
                         added.AlbumRating = (metadata[i].rating*10)
                     added.SortArtist = (metadata[i].author_sort.title())
@@ -1053,20 +1129,6 @@ class ITUNES(DevicePlugin):
                   ub['author']))
         self.log.info()
 
-    def _find_device_book(self, cached_book):
-        '''
-        Windows-only method to get a handle to a device book in the current pythoncom session
-        '''
-        SearchField = ['All','Visible','Artists','Titles','Composers','SongNames']
-        if iswindows:
-            dev_books = self.iTunes.sources.ItemByName(self.sources['iPod']).Playlists.ItemByName('Books')
-            hits = dev_books.Search(cached_book['title'],SearchField.index('Titles'))
-            if hits:
-                for hit in hits:
-                    if hit.Artist == cached_book['author']:
-                        return hit
-            return None
-
     def _find_library_book(self, cached_book):
         '''
         Windows-only method to get a handle to a library book in the current pythoncom session
@@ -1076,7 +1138,28 @@ class ITUNES(DevicePlugin):
             if DEBUG:
                 self.log.info("ITUNES._find_library_book()")
                 self.log.info(" looking for '%s' by %s" % (cached_book['title'], cached_book['author']))
-            lib_books = self.iTunes.sources.ItemByName('Library').Playlists.ItemByName('Books')
+
+            for source in self.iTunes.sources:
+                if source.Kind == self.Sources.index('Library'):
+                    lib = source
+                    if DEBUG:
+                        self.log.info(" Library source: '%s'  kind: %s" % (lib.Name, self.Sources[lib.Kind]))
+                    break
+            else:
+                if DEBUG:
+                    self.log.info(" Library source not found")
+
+            if lib is not None:
+                lib_books = None
+                for pl in lib.Playlists:
+                    if self.PlaylistKind[pl.Kind] == 'User' and self.PlaylistSpecialKind[pl.SpecialKind] == 'Books':
+                        if DEBUG:
+                            self.log.info(" Books playlist: '%s' special_kind: '%s'" % (pl.Name, self.PlaylistSpecialKind[pl.SpecialKind]))
+                        lib_books = pl
+                        break
+                else:
+                    if DEBUG:
+                        self.log.error(" no Books playlist found")
 
             attempts = 9
             while attempts:
@@ -1084,7 +1167,7 @@ class ITUNES(DevicePlugin):
                 hits = lib_books.Search(cached_book['author'],SearchField.index('Artists'))
                 if hits:
                     for hit in hits:
-                        self.log.info(" evaluating '%s' by %s" % (hit.Name, hit.Artist))
+                        #self.log.info(" evaluating '%s' by %s" % (hit.Name, hit.Artist))
                         if hit.Name == cached_book['title']:
                             self.log.info(" matched '%s' by %s" % (hit.Name, hit.Artist))
                             return hit
@@ -1114,12 +1197,6 @@ class ITUNES(DevicePlugin):
         except:
             zfw = zipfile.ZipFile(archive_path, mode='a')
         else:
-#             if DEBUG:
-#                 if isosx:
-#                     self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.name())
-#                 elif iswindows:
-#                     self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.Name)
-
             return thumb_data
 
         if isosx:
@@ -1153,7 +1230,7 @@ class ITUNES(DevicePlugin):
                 return None
 
             # Save the cover from iTunes
-            tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % ArtworkFormat[book.Artwork.Item(1).Format])
+            tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % self.ArtworkFormat[book.Artwork.Item(1).Format])
             book.Artwork.Item(1).SaveArtworkToFile(tmp_thumb)
             try:
                 # Resize the cover
@@ -1174,86 +1251,150 @@ class ITUNES(DevicePlugin):
                 self.log.error(" error generating thumb for '%s'" % book.Name)
                 return None
 
-    def _get_device_book_size(self, title, author):
+    def _get_device_book_size(self, file, compressed_size):
         '''
-        Fetch the size of a book stored on the device
-
-        Windows: If sync-in-progress, this call blocked until sync completes
+        Calculate the exploded size of file
         '''
+        myZip = zipfile.ZipFile(file,'r')
+        myZipList = myZip.infolist()
+        exploded_file_size = 0
+        for file in myZipList:
+            exploded_file_size += file.file_size
         if DEBUG:
-            self.log.info("ITUNES._get_device_book_size():\n looking for title: '%s' author: '%s'" %
-                (title,author))
-
-        device_books = self._get_device_books()
-
-        if isosx:
-            for d_book in device_books:
-                if d_book.name() == title and d_book.artist() == author:
-                    if DEBUG:
-                        self.log.info(' found it')
-                    return d_book.size()
-            else:
-                self.log.error("ITUNES._get_device_book_size():"
-                               " could not find '%s' by '%s' in device_books" % (title,author))
-                return None
-        elif iswindows:
-            for d_book in device_books:
-                if d_book.Name == title and d_book.Artist == author:
-                    self.log.info(" found it")
-                    return d_book.Size
-            else:
-                self.log.error(" could not find '%s' by '%s' in device_books" % (title,author))
-                return None
+            self.log.info("ITUNES._get_device_book_size()")
+            self.log.info(" %d items in archive" % len(myZipList))
+            self.log.info(" compressed: %d  exploded: %d" % (compressed_size, exploded_file_size))
+        return exploded_file_size
 
     def _get_device_books(self):
         '''
-        Assumes pythoncom wrapper
+        Assumes pythoncom wrapper for Windows
         '''
+        if DEBUG:
+            self.log.info("\nITUNES._get_device_books()")
+
+        device_books = []
         if isosx:
             if 'iPod' in self.sources:
                 connected_device = self.sources['iPod']
-                if 'Books' in self.iTunes.sources[connected_device].playlists.name():
-                    return self.iTunes.sources[connected_device].playlists['Books'].file_tracks()
-            return []
+                device = self.iTunes.sources[connected_device]
+                for pl in device.playlists():
+                    if pl.special_kind() == appscript.k.Books:
+                        if DEBUG:
+                            self.log.info(" Book playlist: '%s' special_kind: '%s'" % (pl.name(), pl.special_kind()))
+                        books = pl.file_tracks()
+                        break
+                else:
+                    self.log.error(" book_playlist not found")
+
+                for book in books:
+                    if book.kind() in ['Book','Protected book']:
+                        device_books.append(book)
+                    else:
+                        if DEBUG:
+                            self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind()))
 
         elif iswindows:
             if 'iPod' in self.sources:
-                connected_device = self.sources['iPod']
-                dev = self.iTunes.sources.ItemByName(connected_device)
-                dev_playlists = [pl.Name for pl in dev.Playlists]
-                if 'Books' in dev_playlists:
-                    return self.iTunes.sources.ItemByName(connected_device).Playlists.ItemByName('Books').Tracks
-                else:
-                    return []
-            if DEBUG:
-                self.log.warning('ITUNES._get_device_book(): No iPod device connected')
-            return []
+                try:
+                    pythoncom.CoInitialize()
+                    connected_device = self.sources['iPod']
+                    device = self.iTunes.sources.ItemByName(connected_device)
+
+                    dev_books = None
+                    for pl in device.Playlists:
+                        if self.PlaylistKind[pl.Kind] == 'User' and self.PlaylistSpecialKind[pl.SpecialKind] == 'Books':
+                            if DEBUG:
+                                self.log.info(" Books playlist: '%s' special_kind: '%s'" % (pl.Name, self.PlaylistSpecialKind[pl.SpecialKind]))
+                            dev_books = pl.Tracks
+                            break
+                    else:
+                        if DEBUG:
+                            self.log.info(" no Books playlist found")
+
+                    for book in dev_books:
+                        if book.KindAsString in ['Book','Protected book']:
+                            device_books.append(book)
+                        else:
+                            self.log.info(" ignoring '%s' of type %s" % (book.Name, book.KindAsString))
+
+                finally:
+                    pythoncom.CoUninitialize()
+
+        return device_books
 
     def _get_library_books(self):
         '''
         Populate a dict of paths from iTunes Library|Books
         '''
+        if DEBUG:
+            self.log.info("\nITUNES._get_library_books()")
+
         library_books = {}
+        lib = None
 
         if isosx:
-            lib = self.iTunes.sources['library']
-            if 'Books' in lib.playlists.name():
-                lib_books = lib.playlists['Books'].file_tracks()
+            for source in self.iTunes.sources():
+                if source.kind() == appscript.k.library:
+                    lib = source
+                    if DEBUG:
+                        self.log.info(" Library source: '%s' kind: %s" % (lib.name(), lib.kind()))
+                    break
+            else:
+                if DEBUG:
+                    self.log.error(' Library source not found')
+
+            if lib is not None:
+                lib_books = None
+                for pl in lib.playlists():
+                    if pl.special_kind() == appscript.k.Books:
+                        if DEBUG:
+                            self.log.info(" Books playlist: '%s' special_kind: '%s'" % (pl.name(), pl.special_kind()))
+                        break
+                lib_books = pl.file_tracks()
                 for book in lib_books:
-                    path = self.path_template % (book.name(), book.artist())
-                    library_books[path] = book
+                    if book.kind() in ['Book','Protected book']:
+                        path = self.path_template % (book.name(), book.artist())
+                        library_books[path] = book
+                    else:
+                        if DEBUG:
+                            self.log.info(" ignoring library book of type '%s'" % book.kind())
+            else:
+                if DEBUG:
+                    self.log.info('ITUNES._get_library_books():\n No Books playlist')
 
         elif iswindows:
+            lib = None
             try:
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                lib = self.iTunes.sources.ItemByName('Library')
-                lib_playlists = [pl.Name for pl in lib.Playlists]
-                if 'Books' in lib_playlists:
-                    lib_books = lib.Playlists.ItemByName('Books').Tracks
+                for source in self.iTunes.sources:
+                    if source.Kind == self.Sources.index('Library'):
+                        lib = source
+                        self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind]))
+                        break
+                else:
+                    self.log.error(" Library source not found")
+
+                if lib is not None:
+                    lib_books = None
+                    for pl in lib.Playlists:
+                        if self.PlaylistKind[pl.Kind] == 'User' and self.PlaylistSpecialKind[pl.SpecialKind] == 'Books':
+                            if DEBUG:
+                                self.log.info(" Books playlist: '%s' special_kind: '%s'" % (pl.Name, self.PlaylistSpecialKind[pl.SpecialKind]))
+                            lib_books = pl.Tracks
+                            break
+                    else:
+                        if DEBUG:
+                            self.log.error(" no Books playlist found")
+
                     for book in lib_books:
-                        path = self.path_template % (book.Name, book.Artist)
-                        library_books[path] = book
+                        if book.KindAsString in ['Book','Protected book']:
+                            path = self.path_template % (book.Name, book.Artist)
+                            library_books[path] = book
+                        else:
+                            if DEBUG:
+                                self.log.info(" ignoring '%s' of type %s" % (book.Name, book.KindAsString))
             finally:
                 pythoncom.CoUninitialize()
 
@@ -1324,11 +1465,21 @@ class ITUNES(DevicePlugin):
                 self.iTunes = appscript.app('iTunes')
                 initial_status = 'already running'
 
+            # Read the current storage path for iTunes media
+            cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
+            proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
+            proc.wait()
+            media_dir = os.path.abspath(proc.communicate()[0].strip())
+            if os.path.exists(media_dir):
+                self.iTunes_media = media_dir
+            else:
+                self.log.error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
+
             if DEBUG:
-                self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
+                self.log.info(" [%s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.name(), self.iTunes.version(), initial_status,
                   self.version[0],self.version[1],self.version[2]))
-
+                self.log.info(" iTunes_media: %s" % self.iTunes_media)
         if iswindows:
             '''
             Launch iTunes if not already running
@@ -1340,40 +1491,59 @@ class ITUNES(DevicePlugin):
                 self.iTunes.Windows[0].Minimized = True
             initial_status = 'launched'
 
+            # Read the current storage path for iTunes media from the XML file
+            with open(self.iTunes.LibraryXMLPath, 'r') as xml:
+                soup = BeautifulSoup(xml.read().decode('utf-8'))
+                mf = soup.find('key',text="Music Folder").parent
+                string = mf.findNext('string').renderContents()
+                media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
+                if os.path.exists(media_dir):
+                    self.iTunes_media = media_dir
+                else:
+                    self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath)
+                    self.log.error(" %s" % string.parent.prettify())
+                    self.log.error(" '%s' not found" % media_dir)
+
             if DEBUG:
                 self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.Windows[0].name, self.iTunes.Version, initial_status,
                   self.version[0],self.version[1],self.version[2]))
+                self.log.info(" iTunes_media: %s" % self.iTunes_media)
 
     def _remove_from_iTunes(self, cached_book):
         '''
         iTunes does not delete books from storage when removing from database
+        We only want to delete stored copies if the file is stored in iTunes
+        We don't want to delete files stored outside of iTunes
         '''
         if isosx:
             storage_path = os.path.split(cached_book['lib_book'].location().path)
-            title_storage_path = storage_path[0]
-            if DEBUG:
-                self.log.info("ITUNES._remove_from_iTunes():")
-                self.log.info(" removing title_storage_path: %s" % title_storage_path)
-            try:
-                shutil.rmtree(title_storage_path)
-            except:
-                self.log.info(" '%s' not empty" % title_storage_path)
+            if cached_book['lib_book'].location().path.startswith(self.iTunes_media):
+                title_storage_path = storage_path[0]
+                if DEBUG:
+                    self.log.info("ITUNES._remove_from_iTunes():")
+                    self.log.info(" removing title_storage_path: %s" % title_storage_path)
+                try:
+                    shutil.rmtree(title_storage_path)
+                except:
+                    self.log.info(" '%s' not empty" % title_storage_path)
 
-            # Clean up title/author directories
-            author_storage_path = os.path.split(title_storage_path)[0]
-            self.log.info(" author_storage_path: %s" % author_storage_path)
-            author_files = os.listdir(author_storage_path)
-            if '.DS_Store' in author_files:
-                author_files.pop(author_files.index('.DS_Store'))
-            if not author_files:
-                shutil.rmtree(author_storage_path)
-                if DEBUG:
-                    self.log.info(" removing empty author_storage_path")
+                # Clean up title/author directories
+                author_storage_path = os.path.split(title_storage_path)[0]
+                self.log.info(" author_storage_path: %s" % author_storage_path)
+                author_files = os.listdir(author_storage_path)
+                if '.DS_Store' in author_files:
+                    author_files.pop(author_files.index('.DS_Store'))
+                if not author_files:
+                    shutil.rmtree(author_storage_path)
+                    if DEBUG:
+                        self.log.info(" removing empty author_storage_path")
+                else:
+                    if DEBUG:
+                        self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
+                        self.log.info(" %s" % '\n'.join(author_files))
             else:
-                if DEBUG:
-                    self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
-                    self.log.info(" %s" % '\n'.join(author_files))
+                self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title'])
 
             self.iTunes.delete(cached_book['lib_book'])
 
@@ -1383,26 +1553,34 @@ class ITUNES(DevicePlugin):
             Windows stores the book under a common author directory, so we just delete the .epub
             '''
             if DEBUG:
-                self.log.info("ITUNES._remove_from_iTunes(): '%s'" % cached_book['title'])
+                self.log.info("ITUNES._remove_from_iTunes():\n '%s'" % cached_book['title'])
             book = self._find_library_book(cached_book)
             if book:
-                if DEBUG:
-                    self.log.info("ITUNES._remove_from_iTunes():\n deleting '%s' at %s" %
-                        (cached_book['title'], book.Location))
-                folder = os.path.split(book.Location)[0]
                 path = book.Location
+                storage_path = os.path.split(book.Location)
+                if book.Location.startswith(self.iTunes_media):
+                    if DEBUG:
+                        self.log.info("ITUNES._remove_from_iTunes():")
+                        self.log.info(" removing '%s' at %s" %
+                            (cached_book['title'], path))
+                    try:
+                        os.remove(path)
+                    except:
+                        self.log.warning(" could not find '%s' in iTunes storage" % path)
+                    try:
+                        os.rmdir(storage_path[0])
+                        self.log.info(" removed folder '%s'" % storage_path[0])
+                    except:
+                        self.log.info(" folder '%s' not found or not empty" % storage_path[0])
+
+                    # Delete from iTunes database
+                else:
+                    self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title'])
+
                 book.Delete()
-                try:
-                    os.remove(path)
-                except:
-                    self.log.warning(" could not find '%s' in iTunes storage" % path)
-                try:
-                    os.rmdir(folder)
-                    self.log.info(" removed folder '%s'" % folder)
-                except:
-                    self.log.info(" folder '%s' not found or not empty" % folder)
+
             else:
-                self.log.warning(" could not find '%s' in iTunes storage" % cached_book['title'])
+                self.log.warning(" could not find '%s' in iTunes database" % cached_book['title'])
 
     def _update_device(self, msg='', wait=True):
         '''
@@ -1448,11 +1626,9 @@ class ITUNES(DevicePlugin):
                             sys.stdout.write('\n')
                             sys.stdout.flush()
                             break
-
             finally:
                 pythoncom.CoUninitialize()
 
-
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields:

src/calibre/devices/hanlin/driver.py

@@ -123,5 +123,12 @@ class BOOX(HANLINV3):
     EBOOK_DIR_MAIN = 'MyBooks'
     EBOOK_DIR_CARD_A = 'MyBooks'
 
+    def windows_sort_drives(self, drives):
+        return drives
 
+    def osx_sort_names(self, names):
+        return names
+
+    def linux_swap_drives(self, drives):
+        return drives
 

src/calibre/devices/interface.py

@@ -287,7 +287,9 @@ class DevicePlugin(Plugin):
         This method should raise a L{FreeSpaceError} if there is not enough
         free space on the device. The text of the FreeSpaceError must contain the
         word "card" if C{on_card} is not None otherwise it must contain the word "memory".
-        :files: A list of paths and/or file-like objects.
+        :files: A list of paths and/or file-like objects. If they are paths and
+        the paths point to temporary files, they may have an additional
+        attribute, original_file_path pointing to the originals.
         :names: A list of file names that the books should have
         once uploaded to the device. len(names) == len(files)
         :return: A list of 3-element tuples. The list is meant to be passed

src/calibre/devices/prs500/cli/main.py

@@ -337,7 +337,7 @@ def main():
             dev.touch(args[0])
         elif command == 'test_file':
             parser = OptionParser(usage=("usage: %prog test_file path\n"
-                'Open device, copy file psecified by path to device and '
+                'Open device, copy file specified by path to device and '
                 'then eject device.'))
             options, args = parser.parse_args(args)
             if len(args) != 1:

src/calibre/devices/prs505/driver.py

@@ -6,10 +6,9 @@ __docformat__ = 'restructuredtext en'
 Device driver for the SONY devices
 '''
 
-import os
-import re
+import os, time, re
 
-from calibre.devices.usbms.driver import USBMS
+from calibre.devices.usbms.driver import USBMS, debug_print
 from calibre.devices.prs505 import MEDIA_XML
 from calibre.devices.prs505 import CACHE_XML
 from calibre.devices.prs505.sony_cache import XMLCache
@@ -66,6 +65,41 @@ class PRS505(USBMS):
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
+    def post_open_callback(self):
+
+        def write_cache(prefix):
+            try:
+                cachep = os.path.join(prefix, *(CACHE_XML.split('/')))
+                if not os.path.exists(cachep):
+                    dname = os.path.dirname(cachep)
+                    if not os.path.exists(dname):
+                        try:
+                            os.makedirs(dname, mode=0777)
+                        except:
+                            time.sleep(5)
+                            os.makedirs(dname, mode=0777)
+                    with open(cachep, 'wb') as f:
+                        f.write(u'''<?xml version="1.0" encoding="UTF-8"?>
+                            <cache xmlns="http://www.kinoma.com/FskCache/1">
+                            </cache>
+                            '''.encode('utf8'))
+                return True
+            except:
+                import traceback
+                traceback.print_exc()
+            return False
+
+        # Make sure we don't have the launcher partition
+        # as one of the cards
+
+        if self._card_a_prefix is not None:
+            if not write_cache(self._card_a_prefix):
+                self._card_a_prefix = None
+        if self._card_b_prefix is not None:
+            if not write_cache(self._card_b_prefix):
+                self._card_b_prefix = None
+
+
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
 
@@ -94,26 +128,31 @@ class PRS505(USBMS):
         return XMLCache(paths, prefixes)
 
     def books(self, oncard=None, end_session=True):
+        debug_print('PRS505: starting fetching books for card', oncard)
         bl = USBMS.books(self, oncard=oncard, end_session=end_session)
         c = self.initialize_XML_cache()
         c.update_booklist(bl, {'carda':1, 'cardb':2}.get(oncard, 0))
+        debug_print('PRS505: finished fetching books for card', oncard)
         return bl
 
     def sync_booklists(self, booklists, end_session=True):
+        debug_print('PRS505: started sync_booklists')
         c = self.initialize_XML_cache()
         blists = {}
         for i in c.paths:
             if booklists[i] is not None:
                 blists[i] = booklists[i]
         opts = self.settings()
-        collections = ['series', 'tags']
         if opts.extra_customization:
             collections = [x.strip() for x in
                     opts.extra_customization.split(',')]
-
+        else:
+            collections = []
+        debug_print('PRS505: collection fields:', collections)
         c.update(blists, collections)
         c.write()
 
         USBMS.sync_booklists(self, booklists, end_session=end_session)
+        debug_print('PRS505: finished sync_booklists')
 
 

src/calibre/devices/prs505/sony_cache.py

@@ -14,6 +14,7 @@ from lxml import etree
 
 from calibre import prints, guess_type
 from calibre.devices.errors import DeviceError
+from calibre.devices.usbms.driver import debug_print
 from calibre.constants import DEBUG
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata import authors_to_string, title_sort
@@ -61,7 +62,7 @@ class XMLCache(object):
 
     def __init__(self, paths, prefixes):
         if DEBUG:
-            prints('Building XMLCache...')
+            debug_print('Building XMLCache...')
             pprint(paths)
         self.paths = paths
         self.prefixes = prefixes
@@ -97,16 +98,17 @@ class XMLCache(object):
         self.record_roots[0] = recs[0]
 
         self.detect_namespaces()
+        debug_print('Done building XMLCache...')
 
 
     # Playlist management {{{
     def purge_broken_playlist_items(self, root):
+        id_map = self.build_id_map(root)
         for pl in root.xpath('//*[local-name()="playlist"]'):
             seen = set([])
             for item in list(pl):
                 id_ = item.get('id', None)
-                if id_ is None or id_ in seen or not root.xpath(
-                    '//*[local-name()!="item" and @id="%s"]'%id_):
+                if id_ is None or id_ in seen or id_map.get(id_, None) is None:
                     if DEBUG:
                         if id_ is None:
                             cause = 'invalid id'
@@ -127,7 +129,7 @@ class XMLCache(object):
             for playlist in root.xpath('//*[local-name()="playlist"]'):
                 if len(playlist) == 0 or not playlist.get('title', None):
                     if DEBUG:
-                        prints('Removing playlist id:', playlist.get('id', None),
+                        debug_print('Removing playlist id:', playlist.get('id', None),
                                 playlist.get('title', None))
                     playlist.getparent().remove(playlist)
 
@@ -149,20 +151,25 @@ class XMLCache(object):
                     seen.add(title)
 
     def get_playlist_map(self):
+        debug_print('Start get_playlist_map')
         ans = {}
         self.ensure_unique_playlist_titles()
+        debug_print('after ensure_unique_playlist_titles')
         self.prune_empty_playlists()
+        debug_print('get_playlist_map loop')
         for i, root in self.record_roots.items():
+            debug_print('get_playlist_map loop', i)
+            id_map = self.build_id_map(root)
             ans[i] = []
             for playlist in root.xpath('//*[local-name()="playlist"]'):
                 items = []
                 for item in playlist:
                     id_ = item.get('id', None)
-                    records = root.xpath(
-                        '//*[local-name()="text" and @id="%s"]'%id_)
-                    if records:
-                        items.append(records[0])
+                    record = id_map.get(id_, None)
+                    if record is not None:
+                        items.append(record)
                 ans[i].append((playlist.get('title'), items))
+        debug_print('end get_playlist_map')
         return ans
 
     def get_or_create_playlist(self, bl_idx, title):
@@ -171,7 +178,7 @@ class XMLCache(object):
             if playlist.get('title', None) == title:
                 return playlist
         if DEBUG:
-            prints('Creating playlist:', title)
+            debug_print('Creating playlist:', title)
         ans = root.makeelement('{%s}playlist'%self.namespaces[bl_idx],
                 nsmap=root.nsmap, attrib={
                     'uuid' : uuid(),
@@ -185,7 +192,7 @@ class XMLCache(object):
 
     def fix_ids(self): # {{{
         if DEBUG:
-            prints('Running fix_ids()')
+            debug_print('Running fix_ids()')
 
         def ensure_numeric_ids(root):
             idmap = {}
@@ -198,8 +205,8 @@ class XMLCache(object):
                     idmap[id_] = '-1'
 
             if DEBUG and idmap:
-                prints('Found non numeric ids:')
-                prints(list(idmap.keys()))
+                debug_print('Found non numeric ids:')
+                debug_print(list(idmap.keys()))
             return idmap
 
         def remap_playlist_references(root, idmap):
@@ -210,7 +217,7 @@ class XMLCache(object):
                     if id_ in idmap:
                         item.set('id', idmap[id_])
                         if DEBUG:
-                            prints('Remapping id %s to %s'%(id_, idmap[id_]))
+                            debug_print('Remapping id %s to %s'%(id_, idmap[id_]))
 
         def ensure_media_xml_base_ids(root):
             for num, tag in enumerate(('library', 'watchSpecial')):
@@ -260,6 +267,8 @@ class XMLCache(object):
         last_bl = max(self.roots.keys())
         max_id = self.max_id(self.roots[last_bl])
         self.roots[0].set('nextID', str(max_id+1))
+        debug_print('Finished running fix_ids()')
+
     # }}}
 
     # Update JSON from XML {{{
@@ -267,7 +276,7 @@ class XMLCache(object):
         if bl_index not in self.record_roots:
             return
         if DEBUG:
-            prints('Updating JSON cache:', bl_index)
+            debug_print('Updating JSON cache:', bl_index)
         root = self.record_roots[bl_index]
         pmap = self.get_playlist_map()[bl_index]
         playlist_map = {}
@@ -279,13 +288,14 @@ class XMLCache(object):
                         playlist_map[path] = []
                     playlist_map[path].append(title)
 
+        lpath_map = self.build_lpath_map(root)
         for book in bl:
-            record = self.book_by_lpath(book.lpath, root)
+            record = lpath_map.get(book.lpath, None)
             if record is not None:
                 title = record.get('title', None)
                 if title is not None and title != book.title:
                     if DEBUG:
-                        prints('Renaming title', book.title, 'to', title)
+                        debug_print('Renaming title', book.title, 'to', title)
                     book.title = title
 # We shouldn't do this for Sonys, because the reader strips
 # all but the first author.
@@ -310,20 +320,24 @@ class XMLCache(object):
                 if book.lpath in playlist_map:
                     tags = playlist_map[book.lpath]
                     book.device_collections = tags
+        debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
+        debug_print('Starting update XML from JSON')
         playlist_map = self.get_playlist_map()
 
         for i, booklist in booklists.items():
             if DEBUG:
-                prints('Updating XML Cache:', i)
+                debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
+            lpath_map = self.build_lpath_map(root)
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
-                record = self.book_by_lpath(book.lpath, root)
+#                record = self.book_by_lpath(book.lpath, root)
+                record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
                 self.update_text_record(record, book, path, i)
@@ -337,16 +351,19 @@ class XMLCache(object):
         # This is needed to update device_collections
         for i, booklist in booklists.items():
             self.update_booklist(booklist, i)
+        debug_print('Finished update XML from JSON')
 
     def update_playlists(self, bl_index, root, booklist, playlist_map,
             collections_attributes):
+        debug_print('Starting update_playlists')
         collections = booklist.get_collections(collections_attributes)
+        lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
-            records = [self.book_by_lpath(b.lpath, root) for b in books]
+            records = [lpath_map.get(b.lpath, None) for b in books]
             # Remove any books that were not found, although this
             # *should* never happen
             if DEBUG and None in records:
-                prints('WARNING: Some elements in the JSON cache were not'
+                debug_print('WARNING: Some elements in the JSON cache were not'
                         ' found in the XML cache')
             records = [x for x in records if x is not None]
             for rec in records:
@@ -355,7 +372,7 @@ class XMLCache(object):
             ids = [x.get('id', None) for x in records]
             if None in ids:
                 if DEBUG:
-                    prints('WARNING: Some <text> elements do not have ids')
+                    debug_print('WARNING: Some <text> elements do not have ids')
                     ids = [x for x in ids if x is not None]
 
             playlist = self.get_or_create_playlist(bl_index, category)
@@ -379,20 +396,21 @@ class XMLCache(object):
             title = playlist.get('title', None)
             if title not in collections:
                 if DEBUG:
-                    prints('Deleting playlist:', playlist.get('title', ''))
+                    debug_print('Deleting playlist:', playlist.get('title', ''))
                 playlist.getparent().remove(playlist)
                 continue
             books = collections[title]
-            records = [self.book_by_lpath(b.lpath, root) for b in books]
+            records = [lpath_map.get(b.lpath, None) for b in books]
             records = [x for x in records if x is not None]
             ids = [x.get('id', None) for x in records]
             ids = [x for x in ids if x is not None]
             for item in list(playlist):
                 if item.get('id', None) not in ids:
                     if DEBUG:
-                        prints('Deleting item:', item.get('id', ''),
+                        debug_print('Deleting item:', item.get('id', ''),
                                 'from playlist:', playlist.get('title', ''))
                     playlist.remove(item)
+        debug_print('Finishing update_playlists')
 
     def create_text_record(self, root, bl_id, lpath):
         namespace = self.namespaces[bl_id]
@@ -408,17 +426,13 @@ class XMLCache(object):
         timestamp = os.path.getctime(path)
         date = strftime(timestamp)
         if date != record.get('date', None):
-            if DEBUG:
-                prints('Changing date of', path, 'from',
-                        record.get('date', ''), 'to', date)
-                prints('\tctime', strftime(os.path.getctime(path)))
-                prints('\tmtime', strftime(os.path.getmtime(path)))
             record.set('date', date)
         record.set('size', str(os.stat(path).st_size))
-        record.set('title', book.title)
+        title = book.title if book.title else _('Unknown')
+        record.set('title', title)
         ts = book.title_sort
         if not ts:
-            ts = title_sort(book.title)
+            ts = title_sort(title)
         record.set('titleSorter', ts)
         record.set('author', authors_to_string(book.authors))
         ext = os.path.splitext(path)[1]
@@ -474,12 +488,24 @@ class XMLCache(object):
     # }}}
 
     # Utility methods {{{
+
+    def build_lpath_map(self, root):
+        m = {}
+        for bk in root.xpath('//*[local-name()="text"]'):
+            m[bk.get('path')] = bk
+        return m
+
+    def build_id_map(self, root):
+        m = {}
+        for bk in root.xpath('//*[local-name()="text"]'):
+            m[bk.get('id')] = bk
+        return m
+
     def book_by_lpath(self, lpath, root):
         matches = root.xpath(u'//*[local-name()="text" and @path="%s"]'%lpath)
         if matches:
             return matches[0]
 
-
     def max_id(self, root):
         ans = -1
         for x in root.xpath('//*[@id]'):
@@ -514,10 +540,10 @@ class XMLCache(object):
                         break
                 self.namespaces[i] = ns
 
-        if DEBUG:
-            prints('Found nsmaps:')
-            pprint(self.nsmaps)
-            prints('Found namespaces:')
-            pprint(self.namespaces)
+#        if DEBUG:
+#            debug_print('Found nsmaps:')
+#            pprint(self.nsmaps)
+#            debug_print('Found namespaces:')
+#            pprint(self.namespaces)
     # }}}
 

src/calibre/devices/usbms/books.py

@@ -46,7 +46,8 @@ class Book(MetaInformation):
             self.smart_update(other)
 
     def __eq__(self, other):
-        return self.path == getattr(other, 'path', None)
+        # use lpath because the prefix can change, changing path
+        return self.path == getattr(other, 'lpath', None)
 
     @dynamic_property
     def db_id(self):
@@ -97,13 +98,24 @@ class Book(MetaInformation):
 
 class BookList(_BookList):
 
+    def __init__(self, oncard, prefix, settings):
+        _BookList.__init__(self, oncard, prefix, settings)
+        self._bookmap = {}
+
     def supports_collections(self):
         return False
 
     def add_book(self, book, replace_metadata):
-        if book not in self:
+        try:
+            b = self.index(book)
+        except (ValueError, IndexError):
+            b = None
+        if b is None:
             self.append(book)
             return True
+        if replace_metadata:
+            self[b].smart_update(book)
+            return True
         return False
 
     def remove_book(self, book):
@@ -112,7 +124,6 @@ class BookList(_BookList):
     def get_collections(self):
         return {}
 
-
 class CollectionsBookList(BookList):
 
     def supports_collections(self):

src/calibre/devices/usbms/device.py

@@ -765,12 +765,8 @@ class Device(DeviceConfig, DevicePlugin):
             path = existing[0]
 
         def get_size(obj):
-            if hasattr(obj, 'seek'):
-                obj.seek(0, os.SEEK_END)
-                size = obj.tell()
-                obj.seek(0)
-                return size
-            return os.path.getsize(obj)
+            path = getattr(obj, 'name', obj)
+            return os.path.getsize(path)
 
         sizes = [get_size(f) for f in files]
         size = sum(sizes)

src/calibre/devices/usbms/driver.py

@@ -12,15 +12,24 @@ for a particular device.
 
 import os
 import re
+import time
 import json
 from itertools import cycle
 
 from calibre import prints, isbytestring
-from calibre.constants import filesystem_encoding
+from calibre.constants import filesystem_encoding, DEBUG
 from calibre.devices.usbms.cli import CLI
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
 
+BASE_TIME = None
+def debug_print(*args):
+    global BASE_TIME
+    if BASE_TIME is None:
+        BASE_TIME = time.time()
+    if DEBUG:
+        prints('DEBUG: %6.1f'%(time.time()-BASE_TIME), *args)
+
 # CLI must come before Device as it implements the CLI functions that
 # are inherited from the device interface in Device.
 class USBMS(CLI, Device):
@@ -47,6 +56,8 @@ class USBMS(CLI, Device):
     def books(self, oncard=None, end_session=True):
         from calibre.ebooks.metadata.meta import path_to_ext
 
+        debug_print ('USBMS: Fetching list of books from device. oncard=', oncard)
+
         dummy_bl = BookList(None, None, None)
 
         if oncard == 'carda' and not self._card_a_prefix:
@@ -136,8 +147,8 @@ class USBMS(CLI, Device):
                 need_sync = True
                 del bl[idx]
 
-        #print "count found in cache: %d, count of files in metadata: %d, need_sync: %s" % \
-        #      (len(bl_cache), len(bl), need_sync)
+        debug_print('USBMS: count found in cache: %d, count of files in metadata: %d, need_sync: %s' % \
+            (len(bl_cache), len(bl), need_sync))
         if need_sync: #self.count_found_in_bl != len(bl) or need_sync:
             if oncard == 'cardb':
                 self.sync_booklists((None, None, bl))
@@ -147,10 +158,13 @@ class USBMS(CLI, Device):
                 self.sync_booklists((bl, None, None))
 
         self.report_progress(1.0, _('Getting list of books on device...'))
+        debug_print('USBMS: Finished fetching list of books from device. oncard=', oncard)
         return bl
 
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
+        debug_print('USBMS: uploading %d books'%(len(files)))
+
         path = self._sanity_check(on_card, files)
 
         paths = []
@@ -174,6 +188,7 @@ class USBMS(CLI, Device):
             self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
 
         self.report_progress(1.0, _('Transferring books to device...'))
+        debug_print('USBMS: finished uploading %d books'%(len(files)))
         return zip(paths, cycle([on_card]))
 
     def upload_cover(self, path, filename, metadata):
@@ -186,6 +201,8 @@ class USBMS(CLI, Device):
         pass
 
     def add_books_to_metadata(self, locations, metadata, booklists):
+        debug_print('USBMS: adding metadata for %d books'%(len(metadata)))
+
         metadata = iter(metadata)
         for i, location in enumerate(locations):
             self.report_progress((i+1) / float(len(locations)), _('Adding books to device metadata listing...'))
@@ -218,8 +235,10 @@ class USBMS(CLI, Device):
                 book.size = os.stat(self.normalize_path(path)).st_size
             booklists[blist].add_book(book, replace_metadata=True)
         self.report_progress(1.0, _('Adding books to device metadata listing...'))
+        debug_print('USBMS: finished adding metadata')
 
     def delete_books(self, paths, end_session=True):
+        debug_print('USBMS: deleting %d books'%(len(paths)))
         for i, path in enumerate(paths):
             self.report_progress((i+1) / float(len(paths)), _('Removing books from device...'))
             path = self.normalize_path(path)
@@ -240,8 +259,11 @@ class USBMS(CLI, Device):
                     except:
                         pass
         self.report_progress(1.0, _('Removing books from device...'))
+        debug_print('USBMS: finished deleting %d books'%(len(paths)))
 
     def remove_books_from_metadata(self, paths, booklists):
+        debug_print('USBMS: removing metadata for %d books'%(len(paths)))
+
         for i, path in enumerate(paths):
             self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...'))
             for bl in booklists:
@@ -249,8 +271,11 @@ class USBMS(CLI, Device):
                     if path.endswith(book.path):
                         bl.remove_book(book)
         self.report_progress(1.0, _('Removing books from device metadata listing...'))
+        debug_print('USBMS: finished removing metadata for %d books'%(len(paths)))
 
     def sync_booklists(self, booklists, end_session=True):
+        debug_print('USBMS: starting sync_booklists')
+
         if not os.path.exists(self.normalize_path(self._main_prefix)):
             os.makedirs(self.normalize_path(self._main_prefix))
 
@@ -267,6 +292,7 @@ class USBMS(CLI, Device):
         write_prefix(self._card_b_prefix, 2)
 
         self.report_progress(1.0, _('Sending metadata to device...'))
+        debug_print('USBMS: finished sync_booklists')
 
     @classmethod
     def path_to_unicode(cls, path):

src/calibre/gui2/device.py

@@ -1,6 +1,8 @@
 from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+# Imports {{{
 import os, traceback, Queue, time, socket, cStringIO, re
 from threading import Thread, RLock
 from itertools import repeat
@@ -27,7 +29,9 @@ from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
         config as email_config
 from calibre.devices.folder_device.driver import FOLDER_DEVICE
 
-class DeviceJob(BaseJob):
+# }}}
+
+class DeviceJob(BaseJob): # {{{
 
     def __init__(self, func, done, job_manager, args=[], kwargs={},
             description=''):
@@ -78,8 +82,9 @@ class DeviceJob(BaseJob):
     def log_file(self):
         return cStringIO.StringIO(self._details.encode('utf-8'))
 
+    # }}}
 
-class DeviceManager(Thread):
+class DeviceManager(Thread): # {{{
 
     def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2):
         '''
@@ -122,7 +127,7 @@ class DeviceManager(Thread):
             try:
                 dev.open()
             except:
-                print 'Unable to open device', dev
+                prints('Unable to open device', str(dev))
                 traceback.print_exc()
                 continue
             self.connected_device = dev
@@ -168,11 +173,11 @@ class DeviceManager(Thread):
             if possibly_connected_devices:
                 if not self.do_connect(possibly_connected_devices,
                                        is_folder_device=False):
-                    print 'Connect to device failed, retrying in 5 seconds...'
+                    prints('Connect to device failed, retrying in 5 seconds...')
                     time.sleep(5)
                     if not self.do_connect(possibly_connected_devices,
                                        is_folder_device=False):
-                        print 'Device connect failed again, giving up'
+                        prints('Device connect failed again, giving up')
 
     def umount_device(self, *args):
         if self.is_device_connected and not self.job_manager.has_device_jobs():
@@ -317,7 +322,7 @@ class DeviceManager(Thread):
     def _save_books(self, paths, target):
         '''Copy books from device to disk'''
         for path in paths:
-            name = path.rpartition(getattr(self.device, 'path_sep', '/'))[2]
+            name = path.rpartition(os.sep)[2]
             dest = os.path.join(target, name)
             if os.path.abspath(dest) != os.path.abspath(path):
                 f = open(dest, 'wb')
@@ -338,8 +343,9 @@ class DeviceManager(Thread):
         return self.create_job(self._view_book, done, args=[path, target],
                         description=_('View book on device'))
 
+    # }}}
 
-class DeviceAction(QAction):
+class DeviceAction(QAction): # {{{
 
     a_s = pyqtSignal(object)
 
@@ -356,9 +362,9 @@ class DeviceAction(QAction):
     def __repr__(self):
         return self.__class__.__name__ + ':%s:%s:%s'%(self.dest, self.delete,
                 self.specific)
+    # }}}
 
-
-class DeviceMenu(QMenu):
+class DeviceMenu(QMenu): # {{{
 
     fetch_annotations = pyqtSignal()
     connect_to_folder = pyqtSignal()
@@ -532,8 +538,9 @@ class DeviceMenu(QMenu):
         annot_enable = enable and getattr(device, 'SUPPORTS_ANNOTATIONS', False)
         self.annotation_action.setEnabled(annot_enable)
 
+    # }}}
 
-class Emailer(Thread):
+class Emailer(Thread): # {{{
 
     def __init__(self, timeout=60):
         Thread.__init__(self)
@@ -590,6 +597,7 @@ class Emailer(Thread):
                 results.append([jobname, e, traceback.format_exc()])
         callback(results)
 
+    # }}}
 
 class DeviceGUI(object):
 
@@ -637,7 +645,7 @@ class DeviceGUI(object):
         if not ids or len(ids) == 0:
             return
         files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
-                                    fmts, paths=True, set_metadata=True,
+                                    fmts, set_metadata=True,
                                     specific_format=specific_format,
                                     exclude_auto=do_auto_convert)
         if do_auto_convert:
@@ -647,7 +655,6 @@ class DeviceGUI(object):
             _auto_ids = []
 
         full_metadata = self.library_view.model().metadata_for(ids)
-        files = [getattr(f, 'name', None) for f in files]
 
         bad, remove_ids, jobnames = [], [], []
         texts, subjects, attachments, attachment_names = [], [], [], []
@@ -760,7 +767,7 @@ class DeviceGUI(object):
         for account, fmts in accounts:
             files, auto = self.library_view.model().\
                     get_preferred_formats_from_ids([id], fmts)
-            files = [f.name for f in files if f is not None]
+            files = [f for f in files if f is not None]
             if not files:
                 continue
             attachment = files[0]
@@ -824,7 +831,7 @@ class DeviceGUI(object):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 prefix = ascii_filename(prefix)
                 names.append('%s_%d%s'%(prefix, id,
-                    os.path.splitext(f.name)[1]))
+                    os.path.splitext(f)[1]))
                 if mi.cover and os.access(mi.cover, os.R_OK):
                     mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                         'rb').read())
@@ -837,7 +844,7 @@ class DeviceGUI(object):
                 on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                 self.upload_books(files, names, metadata,
                         on_card=on_card,
-                        memory=[[f.name for f in files], remove])
+                        memory=[files, remove])
                 self.status_bar.showMessage(_('Sending catalogs to device.'), 5000)
 
 
@@ -884,7 +891,7 @@ class DeviceGUI(object):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 prefix = ascii_filename(prefix)
                 names.append('%s_%d%s'%(prefix, id,
-                    os.path.splitext(f.name)[1]))
+                    os.path.splitext(f)[1]))
                 if mi.cover and os.access(mi.cover, os.R_OK):
                     mi.thumbnail = self.cover_to_thumbnail(open(mi.cover,
                         'rb').read())
@@ -898,7 +905,7 @@ class DeviceGUI(object):
                 on_card = space.get(sorted(space.keys(), reverse=True)[0], None)
                 self.upload_books(files, names, metadata,
                         on_card=on_card,
-                        memory=[[f.name for f in files], remove])
+                        memory=[files, remove])
                 self.status_bar.showMessage(_('Sending news to device.'), 5000)
 
 
@@ -914,7 +921,7 @@ class DeviceGUI(object):
 
         _files, _auto_ids = self.library_view.model().get_preferred_formats_from_ids(ids,
                                     settings.format_map,
-                                    paths=True, set_metadata=True,
+                                    set_metadata=True,
                                     specific_format=specific_format,
                                     exclude_auto=do_auto_convert)
         if do_auto_convert:
@@ -930,9 +937,8 @@ class DeviceGUI(object):
                 mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read())
         imetadata = iter(metadata)
 
-        files = [getattr(f, 'name', None) for f in _files]
         bad, good, gf, names, remove_ids = [], [], [], [], []
-        for f in files:
+        for f in _files:
             mi = imetadata.next()
             id = ids.next()
             if f is None:

src/calibre/gui2/device_drivers/configwidget.py

@@ -38,9 +38,10 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
             self.opt_read_metadata.setChecked(self.settings.read_metadata)
         else:
             self.opt_read_metadata.hide()
-        if extra_customization_message and settings.extra_customization:
+        if extra_customization_message:
             self.extra_customization_label.setText(extra_customization_message)
-            self.opt_extra_customization.setText(settings.extra_customization)
+            if settings.extra_customization:
+                self.opt_extra_customization.setText(settings.extra_customization)
         else:
             self.extra_customization_label.setVisible(False)
             self.opt_extra_customization.setVisible(False)

src/calibre/gui2/dialogs/config/__init__.py

@@ -445,6 +445,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         self.username.setText(opts.username)
         self.password.setText(opts.password if opts.password else '')
         self.opt_max_opds_items.setValue(opts.max_opds_items)
+        self.opt_max_opds_ungrouped_items.setValue(opts.max_opds_ungrouped_items)
         self.auto_launch.setChecked(config['autolaunch_server'])
         self.systray_icon.setChecked(config['systray_icon'])
         self.sync_news.setChecked(config['upload_news_to_device'])
@@ -848,6 +849,8 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         sc.set('port', self.port.value())
         sc.set('max_cover', mcs)
         sc.set('max_opds_items', self.opt_max_opds_items.value())
+        sc.set('max_opds_ungrouped_items',
+                self.opt_max_opds_ungrouped_items.value())
         config['delete_news_from_library_on_upload'] = self.delete_news.isChecked()
         config['upload_news_to_device'] = self.sync_news.isChecked()
         config['search_as_you_type'] = self.search_as_you_type.isChecked()

src/calibre/gui2/dialogs/config/config.ui

@@ -892,6 +892,26 @@
                  </property>
                 </widget>
                </item>
+               <item row="6" column="1">
+                <widget class="QSpinBox" name="opt_max_opds_ungrouped_items">
+                 <property name="minimum">
+                  <number>25</number>
+                 </property>
+                 <property name="maximum">
+                  <number>1000000</number>
+                 </property>
+                </widget>
+               </item>
+               <item row="6" column="0">
+                <widget class="QLabel" name="label_16">
+                 <property name="text">
+                  <string>Max. OPDS &amp;ungrouped items:</string>
+                 </property>
+                 <property name="buddy">
+                  <cstring>opt_max_opds_ungrouped_items</cstring>
+                 </property>
+                </widget>
+               </item>
               </layout>
              </item>
              <item>

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -3,6 +3,7 @@ __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to create a new custom column'''
 
+import re
 from functools import partial
 
 from PyQt4.QtCore import SIGNAL
@@ -94,8 +95,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         col = unicode(self.column_name_box.text()).lower()
         if not col:
             return self.simple_error('', _('No lookup name was provided'))
-        if not col.isalnum() or not col[0].isalpha():
-            return self.simple_error('', _('The label must contain only letters and digits, and start with a letter'))
+        if re.match('^\w*$', col) is None or not col[0].isalpha():
+            return self.simple_error('', _('The label must contain only letters, digits and underscores, and start with a letter'))
         col_heading = unicode(self.column_heading_box.text())
         col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
         if col_type == '*text':

src/calibre/gui2/dialogs/metadata_single.py

@@ -403,12 +403,14 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         ag = QCoreApplication.instance().desktop().availableGeometry(self)
         self.cover.MAX_HEIGHT = ag.height()-(25 if (islinux or isfreebsd) else 0)-height_of_rest
         self.cover.MAX_WIDTH = ag.width()-(25 if (islinux or isfreebsd) else 0)-width_of_rest
+        pm = QPixmap()
         if cover:
-            pm = QPixmap()
             pm.loadFromData(cover)
-            if not pm.isNull():
-                self.cover.setPixmap(pm)
+        if pm.isNull():
+            pm = QPixmap(I('book.svg'))
+        else:
             self.cover_data = cover
+        self.cover.setPixmap(pm)
         self.original_series_name = unicode(self.series.text()).strip()
         if len(db.custom_column_label_map) == 0:
             self.central_widget.tabBar().setVisible(False)

src/calibre/gui2/library/models.py

@@ -21,7 +21,8 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
-from calibre import strftime
+from calibre import strftime, isbytestring
+from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
 
 def human_readable(size, precision=1):
@@ -33,6 +34,13 @@ TIME_FMT = '%d %b %Y'
 ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center':
         Qt.AlignHCenter}
 
+class FormatPath(unicode):
+
+    def __new__(cls, path, orig_file_path):
+        ans = unicode.__new__(cls, path)
+        ans.orig_file_path = orig_file_path
+        return ans
+
 class BooksModel(QAbstractTableModel): # {{{
 
     about_to_be_sorted   = pyqtSignal(object, name='aboutToBeSorted')
@@ -213,7 +221,7 @@ class BooksModel(QAbstractTableModel): # {{{
             self.endInsertRows()
             self.count_changed()
 
-    def search(self, text, refinement, reset=True):
+    def search(self, text, reset=True):
         try:
             self.db.search(text)
         except ParseException:
@@ -224,9 +232,10 @@ class BooksModel(QAbstractTableModel): # {{{
             self.clear_caches()
             self.reset()
         if self.last_search:
+            # Do not issue search done for the null search. It is used to clear
+            # the search and count records for restrictions
             self.searched.emit(True)
 
-
     def sort(self, col, order, reset=True):
         if not self.db:
             return
@@ -257,7 +266,7 @@ class BooksModel(QAbstractTableModel): # {{{
         self.sort(col, self.sorted_on[1], reset=reset)
 
     def research(self, reset=True):
-        self.search(self.last_search, False, reset=reset)
+        self.search(self.last_search, reset=reset)
 
     def columnCount(self, parent):
         if parent and parent.isValid():
@@ -378,7 +387,7 @@ class BooksModel(QAbstractTableModel): # {{{
         else:
             return metadata
 
-    def get_preferred_formats_from_ids(self, ids, formats, paths=False,
+    def get_preferred_formats_from_ids(self, ids, formats,
                               set_metadata=False, specific_format=None,
                               exclude_auto=False, mode='r+b'):
         ans = []
@@ -403,12 +412,20 @@ class BooksModel(QAbstractTableModel): # {{{
                     as_file=True)) as src:
                     shutil.copyfileobj(src, pt)
                     pt.flush()
+                    if getattr(src, 'name', None):
+                        pt.orig_file_path = os.path.abspath(src.name)
                 pt.seek(0)
                 if set_metadata:
                     _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True),
                                   format)
-                pt.close() if paths else pt.seek(0)
-                ans.append(pt)
+                pt.close()
+                def to_uni(x):
+                    if isbytestring(x):
+                        x = x.decode(filesystem_encoding)
+                    return x
+                name, op = map(to_uni, map(os.path.abspath, (pt.name,
+                    pt.orig_file_path)))
+                ans.append(FormatPath(name, op))
             else:
                 need_auto.append(id)
                 if not exclude_auto:
@@ -730,6 +747,8 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_search_restriction(self, s):
         self.db.data.set_search_restriction(s)
+        self.search('')
+        return self.rowCount(None)
 
 # }}}
 
@@ -874,7 +893,7 @@ class DeviceBooksModel(BooksModel): # {{{
         return flags
 
 
-    def search(self, text, refinement, reset=True):
+    def search(self, text, reset=True):
         if not text or not text.strip():
             self.map = list(range(len(self.db)))
         else:
@@ -1086,7 +1105,6 @@ class DeviceBooksModel(BooksModel): # {{{
             idx = self.map[row]
             if cname == 'title' :
                 self.db[idx].title = val
-                self.db[idx].title_sorter = val
             elif cname == 'authors':
                 self.db[idx].authors = string_to_authors(val)
             elif cname == 'collections':

src/calibre/gui2/library/views.py

@@ -426,6 +426,14 @@ class BooksView(QTableView): # {{{
         if dy != 0:
             self.column_header.update()
 
+    def scroll_to_row(self, row):
+        if row > -1:
+            h = self.horizontalHeader()
+            for i in range(h.count()):
+                if not h.isSectionHidden(i):
+                    self.scrollTo(self.model().index(row, i))
+                    break
+
     def close(self):
         self._model.close()
 
@@ -437,10 +445,6 @@ class BooksView(QTableView): # {{{
         self._search_done = search_done
         self._model.searched.connect(self.search_done)
 
-    def connect_to_restriction_set(self, tv):
-        # must be synchronous (not queued)
-        tv.restriction_set.connect(self._model.set_search_restriction)
-
     def connect_to_book_display(self, bd):
         self._model.new_bookdisplay_data.connect(bd)
 

src/calibre/gui2/lrf_renderer/main.py

@@ -152,7 +152,7 @@ class Main(MainWindow, Ui_MainWindow):
             self.stack.setCurrentIndex(1)
             self.renderer.start()
 
-    def find(self, search, refinement):
+    def find(self, search):
         self.last_search = search
         try:
             self.document.search(search)

src/calibre/gui2/main.py

@@ -226,7 +226,7 @@ class GuiRunner(QObject):
         self.splash_pixmap = QPixmap()
         self.splash_pixmap.load(I('library.png'))
         self.splash_screen = QSplashScreen(self.splash_pixmap,
-            Qt.SplashScreen|Qt.WindowStaysOnTopHint)
+            Qt.SplashScreen)
         self.splash_screen.showMessage(_('Starting %s: Loading books...') %
                 __appname__)
         self.splash_screen.show()

src/calibre/gui2/main.ui

@@ -28,7 +28,7 @@
     <normaloff>:/images/library.png</normaloff>:/images/library.png</iconset>
   </property>
   <widget class="QWidget" name="centralwidget">
-   <layout class="QVBoxLayout" name="verticalLayout_4">
+   <layout class="QVBoxLayout" name="verticalLayout_2">
     <item>
      <layout class="QHBoxLayout" name="horizontalLayout_3">
       <item>
@@ -305,78 +305,79 @@
      </layout>
     </item>
     <item>
-     <layout class="QHBoxLayout" name="horizontalLayout">
-      <item>
-       <widget class="Splitter" name="vertical_splitter">
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-          <horstretch>0</horstretch>
-          <verstretch>100</verstretch>
-         </sizepolicy>
-        </property>
-        <property name="orientation">
-         <enum>Qt::Vertical</enum>
-        </property>
-        <widget class="QStackedWidget" name="stack">
-         <property name="sizePolicy">
-          <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-           <horstretch>100</horstretch>
-           <verstretch>100</verstretch>
-          </sizepolicy>
-         </property>
-         <property name="currentIndex">
-          <number>0</number>
-         </property>
-         <widget class="QWidget" name="library">
-          <layout class="QVBoxLayout" name="verticalLayout_2">
-           <item>
-            <widget class="Splitter" name="horizontal_splitter">
-             <property name="orientation">
-              <enum>Qt::Horizontal</enum>
-             </property>
-             <widget class="QWidget" name="layoutWidget">
-              <layout class="QVBoxLayout" name="verticalLayout">
-               <item>
-                <widget class="TagsView" name="tags_view">
-                 <property name="tabKeyNavigation">
-                  <bool>true</bool>
-                 </property>
-                 <property name="alternatingRowColors">
-                  <bool>true</bool>
-                 </property>
-                 <property name="animated">
-                  <bool>true</bool>
-                 </property>
-                 <property name="headerHidden">
-                  <bool>true</bool>
-                 </property>
-                </widget>
-               </item>
-               <item>
+     <widget class="Splitter" name="vertical_splitter">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+        <horstretch>0</horstretch>
+        <verstretch>100</verstretch>
+       </sizepolicy>
+      </property>
+      <property name="orientation">
+       <enum>Qt::Vertical</enum>
+      </property>
+      <widget class="QWidget" name="layoutWidget">
+       <layout class="QHBoxLayout" name="horizontalLayout">
+        <item>
+         <widget class="QStackedWidget" name="stack">
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+            <horstretch>100</horstretch>
+            <verstretch>100</verstretch>
+           </sizepolicy>
+          </property>
+          <property name="currentIndex">
+           <number>0</number>
+          </property>
+          <widget class="QWidget" name="library">
+           <layout class="QVBoxLayout" name="verticalLayout_4">
+            <item>
+             <widget class="Splitter" name="horizontal_splitter">
+              <property name="orientation">
+               <enum>Qt::Horizontal</enum>
+              </property>
+              <widget class="QWidget" name="layoutWidget">
+               <layout class="QVBoxLayout" name="verticalLayout">
+                <item>
+                 <widget class="TagsView" name="tags_view">
+                  <property name="tabKeyNavigation">
+                   <bool>true</bool>
+                  </property>
+                  <property name="alternatingRowColors">
+                   <bool>true</bool>
+                  </property>
+                  <property name="animated">
+                   <bool>true</bool>
+                  </property>
+                  <property name="headerHidden">
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
+                <item>
                  <widget class="QCheckBox" name="popularity">
                   <property name="text">
                    <string>Sort by &amp;popularity</string>
                   </property>
                  </widget>
-               </item>
-               <item>
-                <widget class="QComboBox" name="tag_match">
-                 <property name="currentIndex">
-                  <number>0</number>
-                 </property>
-                 <item>
-                  <property name="text">
-                   <string>Match any</string>
+                </item>
+                <item>
+                 <widget class="QComboBox" name="tag_match">
+                  <property name="currentIndex">
+                   <number>0</number>
                   </property>
-                 </item>
-                 <item>
-                  <property name="text">
-                   <string>Match all</string>
-                  </property>
-                 </item>
-                </widget>
-               </item>
-               <item>
+                  <item>
+                   <property name="text">
+                    <string>Match any</string>
+                   </property>
+                  </item>
+                  <item>
+                   <property name="text">
+                    <string>Match all</string>
+                   </property>
+                  </item>
+                 </widget>
+                </item>
+                <item>
                  <widget class="QPushButton" name="edit_categories">
                   <property name="toolTip">
                    <string>Create, edit, and delete user categories</string>
@@ -385,10 +386,55 @@
                    <string>Manage &amp;user categories</string>
                   </property>
                  </widget>
-               </item>
-              </layout>
+                </item>
+               </layout>
+              </widget>
+              <widget class="QWidget" name="">
+               <layout class="QVBoxLayout" name="cb_layout">
+                <item>
+                 <widget class="BooksView" name="library_view">
+                  <property name="sizePolicy">
+                   <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+                    <horstretch>100</horstretch>
+                    <verstretch>10</verstretch>
+                   </sizepolicy>
+                  </property>
+                  <property name="acceptDrops">
+                   <bool>true</bool>
+                  </property>
+                  <property name="dragEnabled">
+                   <bool>true</bool>
+                  </property>
+                  <property name="dragDropOverwriteMode">
+                   <bool>false</bool>
+                  </property>
+                  <property name="dragDropMode">
+                   <enum>QAbstractItemView::DragDrop</enum>
+                  </property>
+                  <property name="alternatingRowColors">
+                   <bool>true</bool>
+                  </property>
+                  <property name="selectionBehavior">
+                   <enum>QAbstractItemView::SelectRows</enum>
+                  </property>
+                  <property name="showGrid">
+                   <bool>false</bool>
+                  </property>
+                  <property name="wordWrap">
+                   <bool>false</bool>
+                  </property>
+                 </widget>
+                </item>
+               </layout>
+              </widget>
              </widget>
-             <widget class="BooksView" name="library_view">
+            </item>
+           </layout>
+          </widget>
+          <widget class="QWidget" name="main_memory">
+           <layout class="QGridLayout">
+            <item row="0" column="0">
+             <widget class="DeviceBooksView" name="memory_view">
               <property name="sizePolicy">
                <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
                 <horstretch>100</horstretch>
@@ -420,139 +466,107 @@
                <bool>false</bool>
               </property>
              </widget>
-            </widget>
-           </item>
-          </layout>
+            </item>
+           </layout>
+          </widget>
+          <widget class="QWidget" name="card_a_memory">
+           <layout class="QGridLayout">
+            <item row="0" column="0">
+             <widget class="DeviceBooksView" name="card_a_view">
+              <property name="sizePolicy">
+               <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+                <horstretch>10</horstretch>
+                <verstretch>10</verstretch>
+               </sizepolicy>
+              </property>
+              <property name="acceptDrops">
+               <bool>true</bool>
+              </property>
+              <property name="dragEnabled">
+               <bool>true</bool>
+              </property>
+              <property name="dragDropOverwriteMode">
+               <bool>false</bool>
+              </property>
+              <property name="dragDropMode">
+               <enum>QAbstractItemView::DragDrop</enum>
+              </property>
+              <property name="alternatingRowColors">
+               <bool>true</bool>
+              </property>
+              <property name="selectionBehavior">
+               <enum>QAbstractItemView::SelectRows</enum>
+              </property>
+              <property name="showGrid">
+               <bool>false</bool>
+              </property>
+              <property name="wordWrap">
+               <bool>false</bool>
+              </property>
+             </widget>
+            </item>
+           </layout>
+          </widget>
+          <widget class="QWidget" name="card_b_memory">
+           <layout class="QGridLayout">
+            <item row="0" column="0">
+             <widget class="DeviceBooksView" name="card_b_view">
+              <property name="sizePolicy">
+               <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+                <horstretch>10</horstretch>
+                <verstretch>10</verstretch>
+               </sizepolicy>
+              </property>
+              <property name="acceptDrops">
+               <bool>true</bool>
+              </property>
+              <property name="dragEnabled">
+               <bool>true</bool>
+              </property>
+              <property name="dragDropOverwriteMode">
+               <bool>false</bool>
+              </property>
+              <property name="dragDropMode">
+               <enum>QAbstractItemView::DragDrop</enum>
+              </property>
+              <property name="alternatingRowColors">
+               <bool>true</bool>
+              </property>
+              <property name="selectionBehavior">
+               <enum>QAbstractItemView::SelectRows</enum>
+              </property>
+              <property name="showGrid">
+               <bool>false</bool>
+              </property>
+              <property name="wordWrap">
+               <bool>false</bool>
+              </property>
+             </widget>
+            </item>
+           </layout>
+          </widget>
          </widget>
-         <widget class="QWidget" name="main_memory">
-          <layout class="QGridLayout">
-           <item row="0" column="0">
-            <widget class="DeviceBooksView" name="memory_view">
-             <property name="sizePolicy">
-              <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-               <horstretch>100</horstretch>
-               <verstretch>10</verstretch>
-              </sizepolicy>
-             </property>
-             <property name="acceptDrops">
-              <bool>true</bool>
-             </property>
-             <property name="dragEnabled">
-              <bool>true</bool>
-             </property>
-             <property name="dragDropOverwriteMode">
-              <bool>false</bool>
-             </property>
-             <property name="dragDropMode">
-              <enum>QAbstractItemView::DragDrop</enum>
-             </property>
-             <property name="alternatingRowColors">
-              <bool>true</bool>
-             </property>
-             <property name="selectionBehavior">
-              <enum>QAbstractItemView::SelectRows</enum>
-             </property>
-             <property name="showGrid">
-              <bool>false</bool>
-             </property>
-             <property name="wordWrap">
-              <bool>false</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
+        </item>
+        <item>
+         <widget class="SideBar" name="sidebar" native="true">
+          <property name="sizePolicy">
+           <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
+            <horstretch>0</horstretch>
+            <verstretch>0</verstretch>
+           </sizepolicy>
+          </property>
+          <property name="minimumSize">
+           <size>
+            <width>30</width>
+            <height>0</height>
+           </size>
+          </property>
          </widget>
-         <widget class="QWidget" name="card_a_memory">
-          <layout class="QGridLayout">
-           <item row="0" column="0">
-            <widget class="DeviceBooksView" name="card_a_view">
-             <property name="sizePolicy">
-              <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
-               <horstretch>10</horstretch>
-               <verstretch>10</verstretch>
-              </sizepolicy>
-             </property>
-             <property name="acceptDrops">
-              <bool>true</bool>
-             </property>
-             <property name="dragEnabled">
-              <bool>true</bool>
-             </property>
-             <property name="dragDropOverwriteMode">
-              <bool>false</bool>
-             </property>
-             <property name="dragDropMode">
-              <enum>QAbstractItemView::DragDrop</enum>
-             </property>
-             <property name="alternatingRowColors">
-              <bool>true</bool>
-             </property>
-             <property name="selectionBehavior">
-              <enum>QAbstractItemView::SelectRows</enum>
-             </property>
-             <property name="showGrid">
-              <bool>false</bool>
-             </property>
-             <property name="wordWrap">
-              <bool>false</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-         <widget class="QWidget" name="card_b_memory">
-          <layout class="QGridLayout">
-           <item row="0" column="0">
-            <widget class="DeviceBooksView" name="card_b_view">
-             <property name="sizePolicy">
-              <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
-               <horstretch>10</horstretch>
-               <verstretch>10</verstretch>
-              </sizepolicy>
-             </property>
-             <property name="acceptDrops">
-              <bool>true</bool>
-             </property>
-             <property name="dragEnabled">
-              <bool>true</bool>
-             </property>
-             <property name="dragDropOverwriteMode">
-              <bool>false</bool>
-             </property>
-             <property name="dragDropMode">
-              <enum>QAbstractItemView::DragDrop</enum>
-             </property>
-             <property name="alternatingRowColors">
-              <bool>true</bool>
-             </property>
-             <property name="selectionBehavior">
-              <enum>QAbstractItemView::SelectRows</enum>
-             </property>
-             <property name="showGrid">
-              <bool>false</bool>
-             </property>
-             <property name="wordWrap">
-              <bool>false</bool>
-             </property>
-            </widget>
-           </item>
-          </layout>
-         </widget>
-        </widget>
-        <widget class="StatusBar" name="status_bar" native="true"/>
-       </widget>
-      </item>
-      <item>
-       <widget class="SideBar" name="sidebar" native="true">
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
-          <horstretch>0</horstretch>
-          <verstretch>0</verstretch>
-         </sizepolicy>
-        </property>
-       </widget>
-      </item>
-     </layout>
+        </item>
+       </layout>
+      </widget>
+      <widget class="StatusBar" name="status_bar" native="true"/>
+     </widget>
     </item>
    </layout>
   </widget>

src/calibre/gui2/search_box.py

@@ -57,7 +57,7 @@ class SearchBox2(QComboBox):
     INTERVAL = 1500 #: Time to wait before emitting search signal
     MAX_COUNT = 25
 
-    search = pyqtSignal(object, object)
+    search = pyqtSignal(object)
 
     def __init__(self, parent=None):
         QComboBox.__init__(self, parent)
@@ -97,8 +97,12 @@ class SearchBox2(QComboBox):
         self.help_state = False
 
     def clear_to_help(self):
+        self.search.emit('')
         self._in_a_search = False
         self.setEditText(self.help_text)
+        if self.timer is not None: # Turn off any timers that got started in setEditText
+            self.killTimer(self.timer)
+            self.timer = None
         self.line_edit.home(False)
         self.help_state = True
         self.line_edit.setStyleSheet(
@@ -111,7 +115,6 @@ class SearchBox2(QComboBox):
 
     def clear(self):
         self.clear_to_help()
-        self.search.emit('', False)
 
     def search_done(self, ok):
         if not unicode(self.currentText()).strip():
@@ -155,9 +158,8 @@ class SearchBox2(QComboBox):
         if not text or text == self.help_text:
             return self.clear()
         self.help_state = False
-        refinement = text.startswith(self.prev_search) and ':' not in text
         self.prev_search = text
-        self.search.emit(text, refinement)
+        self.search.emit(text)
 
         idx = self.findText(text, Qt.MatchFixedString)
         self.block_signals(True)
@@ -187,12 +189,15 @@ class SearchBox2(QComboBox):
         self.set_search_string(joiner.join(tags))
 
     def set_search_string(self, txt):
+        if not txt:
+            self.clear_to_help()
+            return
         self.normalize_state()
         self.setEditText(txt)
         if self.timer is not None: # Turn off any timers that got started in setEditText
             self.killTimer(self.timer)
             self.timer = None
-        self.search.emit(txt, False)
+        self.search.emit(txt)
         self.line_edit.end(False)
         self.initial_state = False
 

src/calibre/gui2/status.py

@@ -52,10 +52,7 @@ class BookInfoDisplay(QWidget):
             QLabel.__init__(self)
             self.setMaximumWidth(81)
             self.setMaximumHeight(108)
-            self.default_pixmap = QPixmap(coverpath).scaled(self.maximumWidth(),
-                                                            self.maximumHeight(),
-                                                            Qt.IgnoreAspectRatio,
-                                                            Qt.SmoothTransformation)
+            self.default_pixmap = QPixmap(coverpath)
             self.setScaledContents(True)
             self.statusbar_height = 120
             self.setPixmap(self.default_pixmap)

src/calibre/gui2/tag_view.py

@@ -22,7 +22,6 @@ from calibre.gui2 import error_dialog
 class TagsView(QTreeView): # {{{
 
     refresh_required    = pyqtSignal()
-    restriction_set     = pyqtSignal(object)
     tags_marked         = pyqtSignal(object, object)
     user_category_edit  = pyqtSignal(object)
     tag_list_edit       = pyqtSignal(object, object)
@@ -37,24 +36,23 @@ class TagsView(QTreeView): # {{{
         self.setIconSize(QSize(30, 30))
         self.tag_match = None
 
-    def set_database(self, db, tag_match, popularity, restriction):
+    def set_database(self, db, tag_match, popularity):
         self.hidden_categories = config['tag_browser_hidden_categories']
         self._model = TagsModel(db, parent=self,
-                                hidden_categories=self.hidden_categories)
+                                hidden_categories=self.hidden_categories,
+                                search_restriction=None)
         self.popularity = popularity
-        self.restriction = restriction
         self.tag_match = tag_match
         self.db = db
+        self.search_restriction = None
         self.setModel(self._model)
         self.setContextMenuPolicy(Qt.CustomContextMenu)
         self.clicked.connect(self.toggle)
         self.customContextMenuRequested.connect(self.show_context_menu)
         self.popularity.setChecked(config['sort_by_popularity'])
         self.popularity.stateChanged.connect(self.sort_changed)
-        self.restriction.activated[str].connect(self.search_restriction_set)
         self.refresh_required.connect(self.recount, type=Qt.QueuedConnection)
         db.add_listener(self.database_changed)
-        self.saved_searches_changed(recount=False)
 
     def database_changed(self, event, ids):
         self.refresh_required.emit()
@@ -65,19 +63,14 @@ class TagsView(QTreeView): # {{{
 
     def sort_changed(self, state):
         config.set('sort_by_popularity', state == Qt.Checked)
-        self.model().refresh()
-        # self.search_restriction_set()
+        self.recount()
 
-    def search_restriction_set(self, s):
-        self.clear()
-        if len(s) == 0:
-            self.search_restriction = ''
+    def set_search_restriction(self, s):
+        if s:
+            self.search_restriction = s
         else:
-            self.search_restriction = 'search:"%s"' % unicode(s).strip()
-        self.model().set_search_restriction(self.search_restriction)
-        self.restriction_set.emit(self.search_restriction)
-        self.recount() # Must happen after the emission of the restriction_set signal
-        self.tags_marked.emit(self._model.tokens(), self.match_all)
+            self.search_restriction = None
+        self.set_new_model()
 
     def mouseReleaseEvent(self, event):
         # Swallow everything except leftButton so context menus work correctly
@@ -144,7 +137,8 @@ class TagsView(QTreeView): # {{{
             # the possibility of renaming that item
             if tag_name and \
                     (key in ['authors', 'tags', 'series', 'publisher', 'search'] or \
-                     self.db.field_metadata[key]['is_custom']):
+                     self.db.field_metadata[key]['is_custom'] and \
+                     self.db.field_metadata[key]['datatype'] != 'rating'):
                 self.context_menu.addAction(_('Rename') + " '" + tag_name + "'",
                         partial(self.context_menu_handler, action='edit_item',
                                 category=tag_item, index=index))
@@ -187,29 +181,24 @@ class TagsView(QTreeView): # {{{
         return True
 
     def clear(self):
-        self.model().clear_state()
+        if self.model():
+            self.model().clear_state()
 
-    def saved_searches_changed(self, recount=True):
-        p = prefs['saved_searches'].keys()
-        p.sort()
-        t = self.restriction.currentText()
-        self.restriction.clear() # rebuild the restrictions combobox using current saved searches
-        self.restriction.addItem('')
-        for s in p:
-            self.restriction.addItem(s)
-        if t in p: # redo the current restriction, if there was one
-            self.restriction.setCurrentIndex(self.restriction.findText(t))
-            self.search_restriction_set(t)
-        if recount:
-            self.recount()
+    def is_visible(self, idx):
+        item = idx.internalPointer()
+        if getattr(item, 'type', None) == TagTreeItem.TAG:
+            idx = idx.parent()
+        return self.isExpanded(idx)
 
     def recount(self, *args):
         ci = self.currentIndex()
         if not ci.isValid():
             ci = self.indexAt(QPoint(10, 10))
-        path = self.model().path_for_index(ci)
+        path = self.model().path_for_index(ci) if self.is_visible(ci) else None
         try:
-            self.model().refresh()
+            if not self.model().refresh(): # categories changed!
+                self.set_new_model()
+                path = None
         except: #Database connection could be closed if an integrity check is happening
             pass
         if path:
@@ -222,9 +211,16 @@ class TagsView(QTreeView): # {{{
     # gone, or if columns have been hidden or restored, we must rebuild the
     # model. Reason: it is much easier than reconstructing the browser tree.
     def set_new_model(self):
-        self._model = TagsModel(self.db, parent=self,
-                                hidden_categories=self.hidden_categories)
-        self.setModel(self._model)
+        try:
+            self._model = TagsModel(self.db, parent=self,
+                                    hidden_categories=self.hidden_categories,
+                                    search_restriction=self.search_restriction)
+            self.setModel(self._model)
+        except:
+            # The DB must be gone. Set the model to None and hope that someone
+            # will call set_database later. I don't know if this in fact works
+            self._model = None
+            self.setModel(None)
     # }}}
 
 class TagTreeItem(object): # {{{
@@ -311,7 +307,7 @@ class TagTreeItem(object): # {{{
 
 class TagsModel(QAbstractItemModel): # {{{
 
-    def __init__(self, db, parent, hidden_categories=None):
+    def __init__(self, db, parent, hidden_categories=None, search_restriction=None):
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
@@ -333,20 +329,10 @@ class TagsModel(QAbstractItemModel): # {{{
         self.db = db
         self.tags_view = parent
         self.hidden_categories = hidden_categories
-        self.search_restriction = ''
-        self.ignore_next_search = 0
-
-        # Reconstruct the user categories, putting them into metadata
-        tb_cats = self.db.field_metadata
-        for k in tb_cats.keys():
-            if tb_cats[k]['kind'] in ['user', 'search']:
-                del tb_cats[k]
-        for user_cat in sorted(prefs['user_categories'].keys()):
-            cat_name = user_cat+':' # add the ':' to avoid name collision
-            tb_cats.add_user_category(label=cat_name, name=user_cat)
-        if len(saved_searches.names()):
-            tb_cats.add_search_category(label='search', name=_('Searches'))
+        self.search_restriction = search_restriction
+        self.row_map = []
 
+        # get_node_tree cannot return None here, because row_map is empty
         data = self.get_node_tree(config['sort_by_popularity'])
         self.root_item = TagTreeItem()
         for i, r in enumerate(self.row_map):
@@ -367,29 +353,44 @@ class TagsModel(QAbstractItemModel): # {{{
         self.search_restriction = s
 
     def get_node_tree(self, sort):
+        old_row_map = self.row_map[:]
         self.row_map = []
         self.categories = []
 
-        if len(self.search_restriction):
-            data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map,
-                        ids=self.db.search(self.search_restriction, return_matches=True))
+        # Reconstruct the user categories, putting them into metadata
+        tb_cats = self.db.field_metadata
+        for k in tb_cats.keys():
+            if tb_cats[k]['kind'] in ['user', 'search']:
+                del tb_cats[k]
+        for user_cat in sorted(prefs['user_categories'].keys()):
+            cat_name = user_cat+':' # add the ':' to avoid name collision
+            tb_cats.add_user_category(label=cat_name, name=user_cat)
+        if len(saved_searches.names()):
+            tb_cats.add_search_category(label='search', name=_('Searches'))
+
+        # Now get the categories
+        if self.search_restriction:
+            data = self.db.get_categories(sort_on_count=sort,
+                        icon_map=self.category_icon_map,
+                        ids=self.db.search('', return_matches=True))
         else:
             data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map)
 
         tb_categories = self.db.field_metadata
-        self.category_items = {}
         for category in tb_categories:
-            if category in data: # They should always be there, but ...
-                # make a map of sets of names per category for duplicate
-                # checking when editing
-                self.category_items[category] = set([tag.name for tag in data[category]])
+            if category in data: # The search category can come and go
                 self.row_map.append(category)
                 self.categories.append(tb_categories[category]['name'])
-
+        if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
+            # A category has been added or removed. We must force a rebuild of
+            # the model
+            return None
         return data
 
     def refresh(self):
         data = self.get_node_tree(config['sort_by_popularity']) # get category data
+        if data is None:
+            return False
         row_index = -1
         for i, r in enumerate(self.row_map):
             if self.hidden_categories and self.categories[i] in self.hidden_categories:
@@ -411,6 +412,7 @@ class TagsModel(QAbstractItemModel): # {{{
                     tag.state = state_map.get(tag.name, 0)
                     t = TagTreeItem(parent=category, data=tag, icon_map=self.icon_state_map)
                 self.endInsertRows()
+        return True
 
     def columnCount(self, parent):
         return 1
@@ -424,6 +426,8 @@ class TagsModel(QAbstractItemModel): # {{{
     def setData(self, index, value, role=Qt.EditRole):
         if not index.isValid():
             return NONE
+        # set up to position at the category label
+        path = self.path_for_index(self.parent(index))
         val = unicode(value.toString())
         if not val:
             error_dialog(self.tags_view, _('Item is blank'),
@@ -431,15 +435,14 @@ class TagsModel(QAbstractItemModel): # {{{
             return False
         item = index.internalPointer()
         key = item.parent.category_key
-        # make certain we know about the category
+        # make certain we know about the item's category
         if key not in self.db.field_metadata:
             return
-        if val in self.category_items[key]:
-            error_dialog(self.tags_view, 'Duplicate item',
-                        _('The name %s is already used.')%val).exec_()
-            return False
-        oldval = item.tag.name
         if key == 'search':
+            if val in saved_searches.names():
+                error_dialog(self.tags_view, _('Duplicate search name'),
+                    _('The saved search name %s is already used.')%val).exec_()
+                return False
             saved_searches.rename(unicode(item.data(role).toString()), val)
             self.tags_view.search_item_renamed.emit()
         else:
@@ -456,10 +459,12 @@ class TagsModel(QAbstractItemModel): # {{{
                                     label=self.db.field_metadata[key]['label'])
             self.tags_view.tag_item_renamed.emit()
         item.tag.name = val
-        self.dataChanged.emit(index, index)
-        # replace the old value in the duplicate detection map with the new one
-        self.category_items[key].discard(oldval)
-        self.category_items[key].add(val)
+        self.refresh() # Should work, because no categories can have disappeared
+        if path:
+            idx = self.index_for_path(path)
+            if idx.isValid():
+                self.tags_view.setCurrentIndex(idx)
+                self.tags_view.scrollTo(idx, QTreeView.PositionAtCenter)
         return True
 
     def headerData(self, *args):
@@ -544,12 +549,6 @@ class TagsModel(QAbstractItemModel): # {{{
     def clear_state(self):
         self.reset_all_states()
 
-    def reinit(self, *args, **kwargs):
-        if self.ignore_next_search == 0:
-            self.reset_all_states()
-        else:
-            self.ignore_next_search -= 1
-
     def toggle(self, index, exclusive):
         if not index.isValid(): return False
         item = index.internalPointer()
@@ -557,7 +556,6 @@ class TagsModel(QAbstractItemModel): # {{{
             item.toggle()
             if exclusive:
                 self.reset_all_states(except_=item.tag)
-            self.ignore_next_search = 2
             self.dataChanged.emit(index, index)
             return True
         return False

src/calibre/gui2/ui.py

@@ -160,9 +160,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.restriction_in_effect = False
         self.search.initialize('main_search_history', colorize=True,
                 help_text=_('Search (For Advanced Search click the button to the left)'))
-        self.connect(self.clear_button, SIGNAL('clicked()'), self.search_clear)
+        self.connect(self.clear_button, SIGNAL('clicked()'), self.search.clear)
         self.connect(self.clear_button, SIGNAL('clicked()'), self.saved_search.clear_to_help)
-        self.search_clear()
+        self.search.clear()
 
         self.saved_search.initialize(saved_searches, self.search, colorize=True,
                 help_text=_('Saved Searches'))
@@ -226,14 +226,14 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.connect(self.quit_action, SIGNAL('triggered(bool)'), self.quit)
         self.connect(self.donate_action, SIGNAL('triggered(bool)'), self.donate)
         self.connect(self.restore_action, SIGNAL('triggered()'),
-                self.show_windows)
+                        self.show_windows)
         self.connect(self.action_show_book_details,
-                SIGNAL('triggered(bool)'), self.show_book_info)
+                     SIGNAL('triggered(bool)'), self.show_book_info)
         self.connect(self.action_restart, SIGNAL('triggered()'),
                      self.restart)
         self.connect(self.system_tray_icon,
-                SIGNAL('activated(QSystemTrayIcon::ActivationReason)'),
-                self.system_tray_icon_activated)
+                     SIGNAL('activated(QSystemTrayIcon::ActivationReason)'),
+                     self.system_tray_icon_activated)
         self.tool_bar.contextMenuEvent = self.no_op
 
         ####################### Start spare job server ########################
@@ -521,8 +521,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                                  self.search_done)),
                              ('connect_to_book_display',
                                  (self.status_bar.book_info.show_data,)),
-                             ('connect_to_restriction_set',
-                                 (self.tags_view,)),
                              ]:
             for view in (self.library_view, self.memory_view, self.card_a_view, self.card_b_view):
                 getattr(view, func)(*args)
@@ -545,24 +543,22 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.cover_cache.start()
         self.library_view.model().cover_cache = self.cover_cache
         self.connect(self.edit_categories, SIGNAL('clicked()'), self.do_user_categories_edit)
-        self.tags_view.set_database(db, self.tag_match, self.popularity, self.search_restriction)
+        self.search_restriction.activated[str].connect(self.apply_search_restriction)
+        self.tags_view.set_database(db, self.tag_match, self.popularity)
         self.tags_view.tags_marked.connect(self.search.search_from_tags)
-        for x in (self.saved_search.clear_to_help, self.mark_restriction_set):
-            self.tags_view.restriction_set.connect(x)
         self.tags_view.tags_marked.connect(self.saved_search.clear_to_help)
         self.tags_view.tag_list_edit.connect(self.do_tags_list_edit)
         self.tags_view.user_category_edit.connect(self.do_user_categories_edit)
         self.tags_view.saved_search_edit.connect(self.do_saved_search_edit)
         self.tags_view.tag_item_renamed.connect(self.do_tag_item_renamed)
         self.tags_view.search_item_renamed.connect(self.saved_search.clear_to_help)
-        self.search.search.connect(self.tags_view.model().reinit)
         for x in (self.location_view.count_changed, self.tags_view.recount,
                 self.restriction_count_changed):
             self.library_view.model().count_changed_signal.connect(x)
 
         self.connect(self.search, SIGNAL('cleared()'), self.search_box_cleared)
-        self.connect(self.saved_search, SIGNAL('changed()'),
-                     self.tags_view.saved_searches_changed, Qt.QueuedConnection)
+        self.connect(self.saved_search, SIGNAL('changed()'), self.saved_searches_changed)
+        self.saved_searches_changed()
         if not gprefs.get('quick_start_guide_added', False):
             from calibre.ebooks.metadata import MetaInformation
             mi = MetaInformation(_('Calibre Quick Start Guide'), ['John Schember'])
@@ -585,7 +581,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.search_restriction.setSizeAdjustPolicy(self.search_restriction.AdjustToMinimumContentsLengthWithIcon)
         self.search_restriction.setMinimumContentsLength(10)
 
-
         ########################### Cover Flow ################################
         self.cover_flow = None
         if CoverFlow is not None:
@@ -602,7 +597,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             self.cover_flow = CoverFlow(height=cfh, text_height=text_height)
             self.cover_flow.setVisible(False)
             if not config['separate_cover_flow']:
-                self.library.layout().addWidget(self.cover_flow)
+                self.cb_layout.addWidget(self.cover_flow)
             self.cover_flow.currentChanged.connect(self.sync_listview_to_cf)
             self.library_view.selectionModel().currentRowChanged.connect(
                     self.sync_cf_to_listview)
@@ -625,7 +620,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 self.sidebar.job_done, Qt.QueuedConnection)
 
 
-
         if config['autolaunch_server']:
             from calibre.library.server.main import start_threaded_server
             from calibre.library.server import server_config
@@ -644,7 +638,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.connect(self.scheduler,
                 SIGNAL('start_recipe_fetch(PyQt_PyObject)'),
                 self.download_scheduled_recipe, Qt.QueuedConnection)
-        self.library_view.verticalHeader().sectionClicked.connect(self.view_specific_book)
 
         for view in ('library', 'memory', 'card_a', 'card_b'):
             view = getattr(self, view+'_view')
@@ -683,7 +676,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         d = SavedSearchEditor(self, search)
         d.exec_()
         if d.result() == d.Accepted:
-            self.tags_view.saved_searches_changed(recount=True)
+            self.saved_searches_changed()
             self.saved_search.clear_to_help()
 
     def resizeEvent(self, ev):
@@ -807,7 +800,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 d.layout().addWidget(self.cover_flow)
                 self.cover_flow.setVisible(True)
                 self.cover_flow.setFocus(Qt.OtherFocusReason)
-                self.library_view.scrollTo(self.library_view.currentIndex())
+                self.library_view.scroll_to_row(self.library_view.currentIndex().row())
                 d.show()
                 d.finished.connect(self.sidebar.external_cover_flow_finished)
                 self.cf_dialog = d
@@ -831,7 +824,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                         self.library_view.currentIndex())
                 self.cover_flow.setVisible(True)
                 self.cover_flow.setFocus(Qt.OtherFocusReason)
-                self.library_view.scrollTo(self.library_view.currentIndex())
+                self.library_view.scroll_to_row(self.library_view.currentIndex().row())
                 self.cover_flow_sync_timer.start(500)
             else:
                 self.cover_flow_sync_timer.stop()
@@ -842,19 +835,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                     sm.select(idx, sm.ClearAndSelect|sm.Rows)
                     self.library_view.setCurrentIndex(idx)
 
-
-
     '''
-    Handling of the count of books in a restricted view requires that
-    we capture the count after the initial restriction search. To so this,
-    we require that the restriction_set signal be issued before the search signal,
-    so that when the search_done happens and the count is displayed,
-    we can grab the count. This works because the search box is cleared
-    when a restriction is set, so that first search will find all books.
-
-    Adding and deleting books creates another complexity. When added, they are
-    displayed regardless of whether they match the restriction. However, if they
-    do not, they are removed at the next search. The counts must take this
+    Restrictions.
+    Adding and deleting books creates a complexity. When added, they are
+    displayed regardless of whether they match a search restriction. However, if
+    they do not, they are removed at the next search. The counts must take this
     behavior into effect.
     '''
 
@@ -862,15 +847,25 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.restriction_count_of_books_in_view += c - self.restriction_count_of_books_in_library
         self.restriction_count_of_books_in_library = c
         if self.restriction_in_effect:
-            self.set_number_of_books_shown(compute_count=False)
+            self.set_number_of_books_shown()
 
-    def mark_restriction_set(self, r):
-        self.restriction_in_effect = False if r is None or not r else True
+    def apply_search_restriction(self, r):
+        r = unicode(r)
+        if r is not None and r != '':
+            self.restriction_in_effect = True
+            restriction = 'search:"%s"'%(r)
+        else:
+            self.restriction_in_effect = False
+            restriction = ''
+        self.restriction_count_of_books_in_view = \
+                    self.library_view.model().set_search_restriction(restriction)
+        self.search.clear_to_help()
+        self.saved_search.clear_to_help()
+        self.tags_view.set_search_restriction(restriction)
+        self.set_number_of_books_shown()
 
-    def set_number_of_books_shown(self, compute_count):
+    def set_number_of_books_shown(self):
         if self.current_view() == self.library_view and self.restriction_in_effect:
-            if compute_count:
-                self.restriction_count_of_books_in_view = self.current_view().row_count()
             t = _("({0} of {1})").format(self.current_view().row_count(),
                                          self.restriction_count_of_books_in_view)
             self.search_count.setStyleSheet('QLabel { border-radius: 8px; background-color: yellow; }')
@@ -884,18 +879,31 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.search_count.setText(t)
 
     def search_box_cleared(self):
-        self.set_number_of_books_shown(compute_count=True)
         self.tags_view.clear()
         self.saved_search.clear_to_help()
-
-    def search_clear(self):
-        self.set_number_of_books_shown(compute_count=True)
-        self.search.clear()
+        self.set_number_of_books_shown()
 
     def search_done(self, view, ok):
         if view is self.current_view():
             self.search.search_done(ok)
-            self.set_number_of_books_shown(compute_count=False)
+            self.set_number_of_books_shown()
+
+    def saved_searches_changed(self):
+        p = prefs['saved_searches'].keys()
+        p.sort()
+        t = unicode(self.search_restriction.currentText())
+        self.search_restriction.clear() # rebuild the restrictions combobox using current saved searches
+        self.search_restriction.addItem('')
+        self.tags_view.recount()
+        for s in p:
+            self.search_restriction.addItem(s)
+        if t:
+            if t in p: # redo the current restriction, if there was one
+                self.search_restriction.setCurrentIndex(self.search_restriction.findText(t))
+                # self.tags_view.set_search_restriction(t)
+            else:
+                self.search_restriction.setCurrentIndex(0)
+                self.apply_search_restriction('')
 
     def sync_cf_to_listview(self, current, previous):
         if self.cover_flow_sync_flag and self.cover_flow.isVisible() and \
@@ -914,6 +922,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 index = m.index(row, 0)
                 if self.library_view.currentIndex().row() != row and index.isValid():
                     self.cover_flow_sync_flag = False
+                    self.library_view.scroll_to_row(index.row())
                     sm = self.library_view.selectionModel()
                     sm.select(index, sm.ClearAndSelect|sm.Rows)
                     self.library_view.setCurrentIndex(index)
@@ -1548,7 +1557,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             if not confirm('<p>'+_('The selected books will be '
                                    '<b>permanently deleted</b> '
                                    'from your device. Are you sure?')
-                                +'</p>', 'library_delete_books', self):
+                                +'</p>', 'device_delete_books', self):
                 return
             if self.stack.currentIndex() == 1:
                 view = self.memory_view
@@ -2304,14 +2313,17 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
     def library_moved(self, newloc):
         if newloc is None: return
         db = LibraryDatabase2(newloc)
+        self.library_path = newloc
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
         self.library_view.set_database(db)
+        self.tags_view.set_database(db, self.tag_match, self.popularity)
         self.library_view.model().set_book_on_device_func(self.book_on_device)
         self.status_bar.clearMessage()
         self.search.clear_to_help()
         self.status_bar.reset_info()
         self.library_view.model().count_changed()
+        prefs['library_path'] = self.library_path
 
     ############################################################################
 
@@ -2358,7 +2370,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             self.search_restriction.setEnabled(False)
             for action in list(self.delete_menu.actions())[1:]:
                 action.setEnabled(False)
-        self.set_number_of_books_shown(compute_count=False)
+        self.set_number_of_books_shown()
 
 
     def device_job_exception(self, job):

src/calibre/gui2/viewer/main.py

@@ -424,7 +424,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.set_bookmarks(self.iterator.bookmarks)
 
 
-    def find(self, text, refinement, repeat=False, backwards=False):
+    def find(self, text, repeat=False, backwards=False):
         if not text:
             self.view.search('')
             return self.search.search_done(False)

src/calibre/library/caches.py

@@ -241,6 +241,24 @@ class ResultCache(SearchQueryParser):
         matches = set([])
         if len(query) < 2:
             return matches
+
+        if location == 'date':
+            location = 'timestamp'
+        loc = self.field_metadata[location]['rec_index']
+
+        if query == 'false':
+            for item in self._data:
+                if item is None: continue
+                if item[loc] is None or item[loc] == UNDEFINED_DATE:
+                    matches.add(item[0])
+            return matches
+        if query == 'true':
+            for item in self._data:
+                if item is None: continue
+                if item[loc] is not None and item[loc] != UNDEFINED_DATE:
+                    matches.add(item[0])
+            return matches
+
         relop = None
         for k in self.date_search_relops.keys():
             if query.startswith(k):
@@ -249,10 +267,6 @@ class ResultCache(SearchQueryParser):
         if relop is None:
                 (p, relop) = self.date_search_relops['=']
 
-        if location == 'date':
-            location = 'timestamp'
-        loc = self.field_metadata[location]['rec_index']
-
         if query == _('today'):
             qd = now()
             field_count = 3
@@ -301,7 +315,7 @@ class ResultCache(SearchQueryParser):
         if query == 'false':
             query = '0'
         elif query == 'true':
-            query = '>0'
+            query = '!=0'
         relop = None
         for k in self.numeric_search_relops.keys():
             if query.startswith(k):

src/calibre/library/custom_columns.py

@@ -183,15 +183,30 @@ class CustomColumns(object):
         ans = self.conn.get('SELECT id, value FROM %s'%table)
         return ans
 
-    def rename_custom_item(self, id, new_name, label=None, num=None):
-        if id:
-            if label is not None:
-                data = self.custom_column_label_map[label]
-            if num is not None:
-                data = self.custom_column_num_map[num]
-            table,lt = self.custom_table_names(data['num'])
-            self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, id))
-            self.conn.commit()
+    def rename_custom_item(self, old_id, new_name, label=None, num=None):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        table,lt = self.custom_table_names(data['num'])
+        # check if item exists
+        new_id = self.conn.get(
+            'SELECT id FROM %s WHERE value=?'%table, (new_name,), all=False)
+        if new_id is None or old_id == new_id:
+            self.conn.execute('UPDATE %s SET value=? WHERE id=?'%table, (new_name, old_id))
+        else:
+            # New id exists. If the column is_multiple, then process like
+            # tags, otherwise process like publishers (see database2)
+            if data['is_multiple']:
+                books = self.conn.get('''SELECT book from %s
+                                         WHERE value=?'''%lt, (old_id,))
+                for (book_id,) in books:
+                    self.conn.execute('''DELETE FROM %s
+                            WHERE book=? and value=?'''%lt, (book_id, new_id))
+            self.conn.execute('''UPDATE %s SET value=?
+                                 WHERE value=?'''%lt, (new_id, old_id,))
+            self.conn.execute('DELETE FROM %s WHERE id=?'%table, (old_id,))
+        self.conn.commit()
 
     def delete_custom_item_using_id(self, id, label=None, num=None):
         if id:

src/calibre/library/database2.py

@@ -9,12 +9,6 @@ The database used to store ebook metadata
 import os, sys, shutil, cStringIO, glob,functools, traceback
 from itertools import repeat
 from math import floor
-try:
-    from PIL import Image as PILImage
-    PILImage
-except ImportError:
-    import Image as PILImage
-
 
 from PyQt4.QtGui import QImage
 
@@ -37,7 +31,7 @@ from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.utils.config import prefs
 from calibre.utils.search_query_parser import saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
-
+from calibre.utils.magick_draw import save_cover_data_to
 
 if iswindows:
     import calibre.utils.winshell as winshell
@@ -475,11 +469,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if callable(getattr(data, 'save', None)):
             data.save(path)
         else:
-            f = data
-            if not callable(getattr(data, 'read', None)):
-                f = cStringIO.StringIO(data)
-            im = PILImage.open(f)
-            im.convert('RGB').save(path, 'JPEG')
+            if callable(getattr(data, 'read', None)):
+                data = data.read()
+            save_cover_data_to(data, path)
 
     def book_on_device(self, id):
         if callable(self.book_on_device_func):
@@ -1007,16 +999,38 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return []
         return result
 
-    def rename_tag(self, id, new_name):
-        if id:
-            self.conn.execute('UPDATE tags SET name=? WHERE id=?', (new_name, id))
-            self.conn.commit()
+    def rename_tag(self, old_id, new_name):
+        new_id = self.conn.get(
+                    '''SELECT id from tags
+                       WHERE name=?''', (new_name,), all=False)
+        if new_id is None or old_id == new_id:
+            # easy cases. Simply rename the tag. Do it even if equal, in case
+            # there is a change of case
+            self.conn.execute('''UPDATE tags SET name=?
+                                 WHERE id=?''', (new_name, old_id))
+        else:
+            # It is possible that by renaming a tag, the tag will appear
+            # twice on a book. This will throw an integrity error, aborting
+            # all the changes. To get around this, we first delete any links
+            # to the new_id from books referencing the old_id, so that
+            # renaming old_id to new_id will be unique on the book
+            books = self.conn.get('''SELECT book from books_tags_link
+                                     WHERE tag=?''', (old_id,))
+            for (book_id,) in books:
+                self.conn.execute('''DELETE FROM books_tags_link
+                                     WHERE book=? and tag=?''', (book_id, new_id))
+
+            # Change the link table to point at the new tag
+            self.conn.execute('''UPDATE books_tags_link SET tag=?
+                                 WHERE tag=?''',(new_id, old_id,))
+            # Get rid of the no-longer used publisher
+            self.conn.execute('DELETE FROM tags WHERE id=?', (old_id,))
+        self.conn.commit()
 
     def delete_tag_using_id(self, id):
-        if id:
-            self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,))
-            self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
-            self.conn.commit()
+        self.conn.execute('DELETE FROM books_tags_link WHERE tag=?', (id,))
+        self.conn.execute('DELETE FROM tags WHERE id=?', (id,))
+        self.conn.commit()
 
     def get_series_with_ids(self):
         result = self.conn.get('SELECT id,name FROM series')
@@ -1024,19 +1038,44 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return []
         return result
 
-    def rename_series(self, id, new_name):
-        if id:
-            self.conn.execute('UPDATE series SET name=? WHERE id=?', (new_name, id))
-            self.conn.commit()
+    def rename_series(self, old_id, new_name):
+        new_id = self.conn.get(
+                    '''SELECT id from series
+                       WHERE name=?''', (new_name,), all=False)
+        if new_id is None or old_id == new_id:
+            self.conn.execute('UPDATE series SET name=? WHERE id=?',
+                              (new_name, old_id))
+        else:
+            # New series exists. Must update the link, then assign a
+            # new series index to each of the books.
+
+            # Get the list of books where we must update the series index
+            books = self.conn.get('''SELECT books.id
+                                     FROM books, books_series_link as lt
+                                     WHERE books.id = lt.book AND lt.series=?
+                                     ORDER BY books.series_index''', (old_id,))
+            # Get the next series index
+            index = self.get_next_series_num_for(new_name)
+            # Now update the link table
+            self.conn.execute('''UPDATE books_series_link
+                                 SET series=?
+                                 WHERE series=?''',(new_id, old_id,))
+            # Now set the indices
+            for (book_id,) in books:
+                self.conn.execute('''UPDATE books
+                                     SET series_index=?
+                                     WHERE id=?''',(index, book_id,))
+                index = index + 1
+        self.conn.commit()
+
 
     def delete_series_using_id(self, id):
-        if id:
-            books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,))
-            self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,))
-            self.conn.execute('DELETE FROM series WHERE id=?', (id,))
-            self.conn.commit()
-            for (book_id,) in books:
-                self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,))
+        books = self.conn.get('SELECT book from books_series_link WHERE series=?', (id,))
+        self.conn.execute('DELETE FROM books_series_link WHERE series=?', (id,))
+        self.conn.execute('DELETE FROM series WHERE id=?', (id,))
+        self.conn.commit()
+        for (book_id,) in books:
+            self.conn.execute('UPDATE books SET series_index=1.0 WHERE id=?', (book_id,))
 
     def get_publishers_with_ids(self):
         result = self.conn.get('SELECT id,name FROM publishers')
@@ -1044,43 +1083,118 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return []
         return result
 
-    def rename_publisher(self, id, new_name):
-        if id:
-            self.conn.execute('UPDATE publishers SET name=? WHERE id=?', (new_name, id))
-            self.conn.commit()
+    def rename_publisher(self, old_id, new_name):
+        new_id = self.conn.get(
+                    '''SELECT id from publishers
+                       WHERE name=?''', (new_name,), all=False)
+        if new_id is None or old_id == new_id:
+            # New name doesn't exist. Simply change the old name
+            self.conn.execute('UPDATE publishers SET name=? WHERE id=?', \
+                              (new_name, old_id))
+        else:
+            # Change the link table to point at the new one
+            self.conn.execute('''UPDATE books_publishers_link
+                                 SET publisher=?
+                                 WHERE publisher=?''',(new_id, old_id,))
+            # Get rid of the no-longer used publisher
+            self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
+        self.conn.commit()
 
-    def delete_publisher_using_id(self, id):
-        if id:
-            self.conn.execute('DELETE FROM books_publishers_link WHERE publisher=?', (id,))
-            self.conn.execute('DELETE FROM publishers WHERE id=?', (id,))
-            self.conn.commit()
+    def delete_publisher_using_id(self, old_id):
+        self.conn.execute('''DELETE FROM books_publishers_link
+                             WHERE publisher=?''', (old_id,))
+        self.conn.execute('DELETE FROM publishers WHERE id=?', (old_id,))
+        self.conn.commit()
 
     # There is no editor for author, so we do not need get_authors_with_ids or
     # delete_author_using_id.
-    def rename_author(self, id, new_name):
-        if id:
-            # Make sure that any commas in new_name are changed to '|'!
-            new_name = new_name.replace(',', '|')
-            self.conn.execute('UPDATE authors SET name=? WHERE id=?', (new_name, id))
-            self.conn.commit()
-            # now must fix up the books
-            books = self.conn.get('SELECT book from books_authors_link WHERE author=?', (id,))
-            for (book_id,) in books:
-                # First, must refresh the cache to see the new authors
-                self.data.refresh_ids(self, [book_id])
-                # now fix the filesystem paths
-                self.set_path(book_id, index_is_id=True)
-                # Next fix the author sort. Reset it to the default
+
+    def rename_author(self, old_id, new_name):
+        # Make sure that any commas in new_name are changed to '|'!
+        new_name = new_name.replace(',', '|')
+
+        # Get the list of books we must fix up, one way or the other
+        # Save the list so we can use it twice
+        bks = self.conn.get('SELECT book from books_authors_link WHERE author=?', (old_id,))
+        books = []
+        for (book_id,) in bks:
+            books.append(book_id)
+
+        # check if the new author already exists
+        new_id = self.conn.get('SELECT id from authors WHERE name=?',
+                                (new_name,), all=False)
+        if new_id is None or old_id == new_id:
+            # No name clash. Go ahead and update the author's name
+            self.conn.execute('UPDATE authors SET name=? WHERE id=?',
+                              (new_name, old_id))
+        else:
+            # First check for the degenerate case -- changing a value to itself.
+            # Update it in case there is a change of case, but do nothing else
+            if old_id == new_id:
+                self.conn.execute('UPDATE authors SET name=? WHERE id=?',
+                              (new_name, old_id))
+                self.conn.commit()
+                return
+            # Author exists. To fix this, we must replace all the authors
+            # instead of replacing the one. Reason: db integrity checks can stop
+            # the rename process, which would leave everything half-done. We
+            # can't do it the same way as tags (delete and add) because author
+            # order is important.
+
+            for book_id in books:
+                # Get the existing list of authors
                 authors = self.conn.get('''
-                    SELECT authors.name
-                    FROM authors, books_authors_link as bl
-                    WHERE bl.book = ? and bl.author = authors.id
-                ''' , (book_id,))
-                # unpack the double-list structure
+                    SELECT author from books_authors_link
+                    WHERE book=?
+                    ORDER BY id''',(book_id,))
+
+                # unpack the double-list structure, replacing the old author
+                # with the new one while we are at it
                 for i,aut in enumerate(authors):
-                    authors[i] = aut[0]
-                ss = authors_to_sort_string(authors)
-                self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (ss, id))
+                    authors[i] = aut[0] if aut[0] != old_id else new_id
+                # Delete the existing authors list
+                self.conn.execute('''DELETE FROM books_authors_link
+                                     WHERE book=?''',(book_id,))
+                # Change the authors to the new list
+                for aid in authors:
+                    try:
+                        self.conn.execute('''
+                            INSERT INTO books_authors_link(book, author)
+                            VALUES (?,?)''', (book_id, aid))
+                    except IntegrityError:
+                        # Sometimes books specify the same author twice in their
+                        # metadata. Ignore it.
+                        pass
+            # Now delete the old author from the DB
+            bks = self.conn.get('SELECT book FROM books_authors_link WHERE author=?', (old_id,))
+            self.conn.execute('DELETE FROM authors WHERE id=?', (old_id,))
+        self.conn.commit()
+        # the authors are now changed, either by changing the author's name
+        # or replacing the author in the list. Now must fix up the books.
+        for book_id in books:
+            # First, must refresh the cache to see the new authors
+            self.data.refresh_ids(self, [book_id])
+            # now fix the filesystem paths
+            self.set_path(book_id, index_is_id=True)
+            # Next fix the author sort. Reset it to the default
+            authors = self.conn.get('''
+                SELECT authors.name
+                FROM authors, books_authors_link as bl
+                WHERE bl.book = ? and bl.author = authors.id
+                ORDER BY bl.id
+            ''' , (book_id,))
+            # unpack the double-list structure
+            for i,aut in enumerate(authors):
+                authors[i] = aut[0]
+            ss = authors_to_sort_string(authors)
+            # Change the '|'s to ','
+            ss = ss.replace('|', ',')
+            self.conn.execute('''UPDATE books
+                                 SET author_sort=?
+                                 WHERE id=?''', (ss, book_id))
+            self.conn.commit()
+            # the caller will do a general refresh, so we don't need to
+            # do one here
 
     # end convenience methods
 

src/calibre/library/server/__init__.py

@@ -38,6 +38,12 @@ def server_config(defaults=None):
     c.add_opt('max_opds_items', ['--max-opds-items'], default=30,
             help=_('The maximum number of matches to return per OPDS query. '
             'This affects Stanza, WordPlayer, etc. integration.'))
+    c.add_opt('max_opds_ungrouped_items', ['--max-opds-ungrouped-items'],
+            default=100,
+            help=_('Group items in categories such as author/tags '
+                'by first letter when there are more than this number '
+                'of items. Default: %default. Set to a large number '
+                'to disable grouping.'))
     return c
 
 def main():

src/calibre/library/server/content.py

@@ -127,10 +127,7 @@ class ContentServer(object):
             cherrypy.log('User agent: '+ua)
 
         if want_opds:
-            return self.stanza(search=kwargs.get('search', None), sortby=kwargs.get('sortby',None), authorid=kwargs.get('authorid',None),
-                           tagid=kwargs.get('tagid',None),
-                           seriesid=kwargs.get('seriesid',None),
-                           offset=kwargs.get('offset', 0))
+            return self.opds(version=0)
 
         if want_mobile:
             return self.mobile()

src/calibre/library/server/opds.py

@@ -445,7 +445,7 @@ class OPDSServer(object):
 
         id_ = 'calibre-category-feed:'+which
 
-        MAX_ITEMS = 50
+        MAX_ITEMS = self.opts.max_opds_ungrouped_items
 
         if len(items) <= MAX_ITEMS:
             max_items = self.opts.max_opds_items
@@ -459,8 +459,6 @@ class OPDSServer(object):
                     self.text, self.count = text, count
 
             starts = set([x.name[0] for x in items])
-            if len(starts) > MAX_ITEMS:
-                starts = set([x.name[:2] for x in items])
             category_groups = OrderedDict()
             for x in sorted(starts, cmp=lambda x,y:cmp(x.lower(), y.lower())):
                 category_groups[x] = len([y for y in items if

src/calibre/manual/customize.rst

@@ -8,16 +8,33 @@ Customizing |app|
 ==================================
 
 |app| has a highly modular design. Various parts of it can be customized. You can learn how to create
-*recipes* to add new sources of online content to |app| in the Section :ref:`news`. Here, you will learn how to
-use *plugins* to customize and control various aspects of |app|'s behavior. 
-
-Theer are different kinds of plugins, corresponding to different aspects of |app|. As more and more aspects of |app|
-are modularized, new plugin types will be added.
+*recipes* to add new sources of online content to |app| in the Section :ref:`news`. Here, you will learn,
+first, how to use environment variables and *tweaks* to customize |app|'s behavior and then how to
+use *plugins* to add funtionality to |app|.
 
 .. contents::
     :depth: 2
     :local:
 
+Environment variables
+-----------------------
+
+    * ``CALIBRE_CONFIG_DIRECTORY`` - sets the directory where configuration files are stored/read.
+    * ``CALIBRE_OVERRIDE_DATABASE_PATH`` - allows you to specify the full path to metadata.db. Using this variable you can have metadata.db be in a location other than the library folder. Useful if your library folder is on a networked drive that does not support file locking.
+    * ``CALIBRE_DEVELOP_FROM`` - Used to run from a calibre development environment. See :ref:`develop`.
+    * ``CALIBRE_OVERRIDE_LANG`` - Used to force the language used by the interface (ISO 639 language code)
+    * ``SYSFS_PATH`` - Use if sysfs is mounted somewhere other than /sys
+    * ``http_proxy`` - Used on linux to specify an HTTP proxy
+
+Tweaks
+------------
+
+Tweaks are small changes that you can specify to control various aspects of |app|'s behavior. You specify them by editing the 2tweaks.py file in the config directory.
+The default tweaks.py file is reproduced below
+
+.. literalinclude:: ../../../resources/default_tweaks.py
+
+
 A Hello World plugin
 ------------------------
 

src/calibre/manual/faq.rst

@@ -157,7 +157,9 @@ If you get timeout errors while browsing the calibre catalog in Stanza, try incr
 Alternative for the iPad
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-As of |app| version 0.7.0, on windows and OS X you can plugin your iPad into the computer using its charging cable, and |app| will detect it and show you a list of books on the iPad. You can then use the Send to device button to send books directly to iBooks on the iPad.
+As of |app| version 0.7.0, you can plugin your iPad into the computer using its charging cable, and |app| will detect it and show you a list of books on the iPad. You can then use the Send to device button to send books directly to iBooks on the iPad.
+
+This method only works on Windows XP and higher and OS X 10.5 and higher. Linux is not supported (iTunes is not available in linux) and OS X 10.4 is not supported.
 
 How do I use |app| with my Android phone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/magick_draw.py

@@ -171,8 +171,7 @@ def add_borders_to_image(path_to_image, left=0, top=0, right=0, bottom=0,
                 border_color)
         compose_image(canvas, img, left, top)
         p.DestroyMagickWand(img)
-        with open(path_to_image, 'wb') as f:
-            p.MagickWriteImage(canvas, f)
+        p.MagickWriteImage(canvas,path_to_image)
         p.DestroyMagickWand(canvas)
 
 def create_cover_page(top_lines, logo_path, width=590, height=750,
@@ -212,6 +211,23 @@ def create_cover_page(top_lines, logo_path, width=590, height=750,
         p.DestroyMagickWand(canvas)
     return ans
 
+def save_cover_data_to(data, path, bgcolor='white'):
+    '''
+    Saves image in data to path, in the format specified by the path
+    extension. Composes the image onto a blank cancas so as to
+    properly convert transparent images.
+    '''
+    with open(path, 'wb') as f:
+        f.write(data)
+    with p.ImageMagick():
+        img = load_image(path)
+        canvas = create_canvas(p.MagickGetImageWidth(img),
+                p.MagickGetImageHeight(img), bgcolor)
+        compose_image(canvas, img, 0, 0)
+        p.MagickWriteImage(canvas, path)
+        p.DestroyMagickWand(img)
+        p.DestroyMagickWand(canvas)
+
 def test():
     import subprocess
     with TemporaryFile('.png') as f:

src/calibre/web/feeds/__init__.py

@@ -11,7 +11,7 @@ from lxml import html
 
 from calibre.web.feeds.feedparser import parse
 from calibre.utils.logging import default_log
-from calibre import entity_to_unicode
+from calibre import entity_to_unicode, strftime
 from calibre.utils.date import dt_factory, utcnow, local_tz
 
 class Article(object):
@@ -53,12 +53,17 @@ class Article(object):
 
     @dynamic_property
     def formatted_date(self):
+
         def fget(self):
             if self._formatted_date is None:
-                self._formatted_date = self.localtime.strftime(" [%a, %d %b %H:%M]")
+                self._formatted_date = strftime(" [%a, %d %b %H:%M]",
+                        t=self.localtime.timetuple())
             return self._formatted_date
+
         def fset(self, val):
-            self._formatted_date = val
+            if isinstance(val, unicode):
+                self._formatted_date = val
+
         return property(fget=fget, fset=fset)
 
     @dynamic_property

src/calibre/web/feeds/news.py

@@ -24,6 +24,7 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.web.feeds import feed_from_xml, templates, feeds_from_index, Feed
 from calibre.web.fetch.simple import option_parser as web2disk_option_parser
 from calibre.web.fetch.simple import RecursiveFetcher
+from calibre.utils.magick_draw import add_borders_to_image
 from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.date import now as nowf
@@ -283,6 +284,15 @@ class BasicNewsRecipe(Recipe):
     #: Override this in your recipe to provide a url to use as a masthead.
     masthead_url = None
 
+    #: By default, the cover image returned by get_cover_url() will be used as
+    #: the cover for the periodical.  Overriding this in your recipe instructs
+    #: calibre to render the downloaded cover into a frame whose width and height
+    #: are expressed as a percentage of the downloaded cover.
+    #: cover_margins = (10,15,'white') pads the cover with a white margin
+    #: 10px on the left and right, 15px on the top and bottom.
+    #: Colors name defined at http://www.imagemagick.org/script/color.php
+    cover_margins = (0,0,'white')
+
     #: Set to a non empty string to disable this recipe
     #: The string will be used as the disabled message
     recipe_disabled = None
@@ -974,6 +984,11 @@ class BasicNewsRecipe(Recipe):
                 self.report_progress(1, _('Downloading cover from %s')%cu)
                 with nested(open(cpath, 'wb'), closing(self.browser.open(cu))) as (cfile, r):
                     cfile.write(r.read())
+                if self.cover_margins[0] or self.cover_margins[1]:
+                    add_borders_to_image(cpath,
+                                         left=self.cover_margins[0],right=self.cover_margins[0],
+                                         top=self.cover_margins[1],bottom=self.cover_margins[1],
+                                         border_color=self.cover_margins[2])
             if ext.lower() == 'pdf':
                 from calibre.ebooks.metadata.pdf import get_metadata
                 stream = open(cpath, 'rb')

src/calibre/web/feeds/recipes/model.py

@@ -241,7 +241,7 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
                 results.add(urn)
         return results
 
-    def search(self, query, refinement):
+    def search(self, query):
         try:
             results = self.parse(unicode(query))
             if not results: