Merge from trunk

recipes/le_temps.recipe

@@ -14,7 +14,7 @@ class LeTemps(BasicNewsRecipe):
      title          = u'Le Temps'
      oldest_article = 7
      max_articles_per_feed = 100
-     __author__ = 'Sujata Raman'
+     __author__ = 'Kovid Goyal'
      description = 'French news. Needs a subscription from http://www.letemps.ch'
      no_stylesheets = True
      remove_javascript = True
@@ -27,6 +27,7 @@ class LeTemps(BasicNewsRecipe):
      def get_browser(self):
          br = BasicNewsRecipe.get_browser(self)
          br.open('http://www.letemps.ch/login')
+         br.select_form(nr=1)
          br['username'] = self.username
          br['password'] = self.password
          raw = br.submit().read()

src/calibre/customize/builtins.py

@@ -875,7 +875,7 @@ class ActionCopyToLibrary(InterfaceActionBase):
 class ActionTweakEpub(InterfaceActionBase):
     name = 'Tweak ePub'
     actual_plugin = 'calibre.gui2.actions.tweak_epub:TweakEpubAction'
-    description = _('Make small twekas to epub files in your calibre library')
+    description = _('Make small tweaks to epub files in your calibre library')
 
 class ActionNextMatch(InterfaceActionBase):
     name = 'Next Match'

src/calibre/customize/conversion.py

@@ -261,7 +261,7 @@ class OutputFormatPlugin(Plugin):
 
     @property
     def description(self):
-        return _('Convert ebooks to the %s format'%self.file_type)
+        return _('Convert ebooks to the %s format')%self.file_type
 
     def __init__(self, *args):
         Plugin.__init__(self, *args)

src/calibre/devices/apple/driver.py

@@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, ctypes, datetime, os, re, shutil, subprocess, sys, tempfile, time
+import cStringIO, ctypes, datetime, os, re, sys, tempfile, time
 from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image, confirm_config_name
 from calibre.constants import isosx, iswindows
@@ -13,8 +13,7 @@ from calibre.devices.errors import OpenFeedback, UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import authors_to_string, MetaInformation, \
-    title_sort
+from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
@@ -165,8 +164,12 @@ class ITUNES(DriverBase):
         settings()
         set_progress_reporter()
         upload_books()
-         _get_fpath()
-          _update_epub_metadata()
+         _remove_existing_copy()
+          _remove_from_device()
+          _remove_from_iTunes()
+         _add_new_copy()
+          _add_library_book()
+         _update_iTunes_metadata()
         add_books_to_metadata()
         use_plugboard_ext()
         set_plugboard()
@@ -183,7 +186,7 @@ class ITUNES(DriverBase):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (1,0,0)
+    version        = (1,1,0)
 
     DISPLAY_DISABLE_DIALOG = "display_disable_apple_driver_dialog"
 
@@ -278,7 +281,6 @@ class ITUNES(DriverBase):
     description_prefix = "added by calibre"
     ejected = False
     iTunes= None
-    iTunes_media = None
     library_orphans = None
     log = Log()
     manual_sync_mode = False
@@ -414,11 +416,11 @@ class ITUNES(DriverBase):
                             this_book.datetime = parse_date(str(book.date_added())).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                         this_book.size = book.size()
                         this_book.uuid = book.composer()
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -453,10 +455,10 @@ class ITUNES(DriverBase):
                                 this_book.datetime = parse_date(str(book.DateAdded)).timetuple()
                             except:
                                 this_book.datetime = time.gmtime()
-                            this_book.db_id = None
                             this_book.device_collections = []
                             this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                             this_book.size = book.Size
+                            this_book.cid = None
                             # Hack to discover if we're running in GUI environment
                             if self.report_progress is not None:
                                 this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -492,7 +494,7 @@ class ITUNES(DriverBase):
 
     def can_handle(self, device_info, debug=False):
         '''
-        Unix version of :method:`can_handle_windows`
+        OSX version of :method:`can_handle_windows`
 
         :param device_info: Is a tupe of (vid, pid, bcd, manufacturer, product,
         serial number)
@@ -1022,17 +1024,14 @@ class ITUNES(DriverBase):
 
         if DEBUG:
             self.log.info("ITUNES.upload_books()")
-            self._dump_files(files, header='upload_books()',indent=2)
-            self._dump_update_list(header='upload_books()',indent=2)
 
         if isosx:
-            for (i,file) in enumerate(files):
-                format = file.rpartition('.')[2].lower()
+            for (i,fpath) in enumerate(files):
+                format = fpath.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title,
                                              authors_to_string(metadata[i].authors),
                                              format)
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
                 thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
                 this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
@@ -1063,13 +1062,12 @@ class ITUNES(DriverBase):
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
-                for (i,file) in enumerate(files):
-                    format = file.rpartition('.')[2].lower()
+                for (i,fpath) in enumerate(files):
+                    format = fpath.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title,
                                                  authors_to_string(metadata[i].authors),
                                                  format)
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -1213,7 +1211,8 @@ class ITUNES(DriverBase):
         '''
         windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._add_library_book()")
+        if DEBUG:
+            self.log.info(" ITUNES._add_library_book()")
         if isosx:
             added = self.iTunes.add(appscript.mactypes.File(file))
 
@@ -1276,24 +1275,59 @@ class ITUNES(DriverBase):
 
     def _add_new_copy(self, fpath, metadata):
         '''
+        fp = cached_book['lib_book'].location().path
+        fp = cached_book['lib_book'].Location
         '''
         if DEBUG:
             self.log.info(" ITUNES._add_new_copy()")
 
+        def _save_last_known_iTunes_storage(lb_added):
+            if isosx:
+                fp = lb_added.location().path
+                index = fp.rfind('/Books') + len('/Books')
+                last_known_iTunes_storage = fp[:index]
+            elif iswindows:
+                fp = lb_added.Location
+                index = fp.rfind('\Books') + len('\Books')
+                last_known_iTunes_storage = fp[:index]
+            dynamic['last_known_iTunes_storage'] = last_known_iTunes_storage
+            self.log.warning("  last_known_iTunes_storage: %s" % last_known_iTunes_storage)
+
         db_added = None
         lb_added = None
 
         if self.manual_sync_mode:
+            '''
+            This is the unsupported direct-connect mode.
+            In an attempt to avoid resetting the iTunes library Media folder, don't try to
+            add the book to iTunes if the last_known_iTunes_storage path is inaccessible.
+            This means that the path has to be set at least once, probably by using
+            'Connect to iTunes' and doing a transfer.
+            '''
+            self.log.warning("  unsupported direct connect mode")
             db_added = self._add_device_book(fpath, metadata)
-            if not getattr(fpath, 'deleted_after_upload', False):
-                lb_added = self._add_library_book(fpath, metadata)
-                if lb_added:
+            last_known_iTunes_storage = dynamic.get('last_known_iTunes_storage', None)
+            if last_known_iTunes_storage is not None:
+                if os.path.exists(last_known_iTunes_storage):
                     if DEBUG:
-                        self.log.info("  file added to Library|Books for iTunes<->iBooks tracking")
+                        self.log.warning("  iTunes storage online, adding to library")
+                    lb_added = self._add_library_book(fpath, metadata)
+                else:
+                    if DEBUG:
+                        self.log.warning("  iTunes storage not online, can't add to library")
+
+            if lb_added:
+                _save_last_known_iTunes_storage(lb_added)
+            if not lb_added and DEBUG:
+                self.log.warn("  failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title)
         else:
             lb_added = self._add_library_book(fpath, metadata)
-            if DEBUG:
-                self.log.info("  file added to Library|Books for pending sync")
+            if lb_added:
+                _save_last_known_iTunes_storage(lb_added)
+            else:
+                raise UserFeedback("iTunes Media folder inaccessible",
+                                   details="Failed to add '%s' to iTunes" % metadata.title,
+                                   level=UserFeedback.WARN)
 
         return db_added, lb_added
 
@@ -1302,14 +1336,17 @@ class ITUNES(DriverBase):
         assumes pythoncom wrapper for db_added
         as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
-        self.log.info(" ITUNES._cover_to_thumb()")
+        if DEBUG:
+            self.log.info(" ITUNES._cover_to_thumb()")
 
         thumb = None
         if metadata.cover:
 
             if format == 'epub':
-                # Pre-shrink cover
-                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                '''
+                Pre-shrink cover
+                self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                '''
                 try:
                     img = PILImage.open(metadata.cover)
                     width = img.size[0]
@@ -1317,8 +1354,8 @@ class ITUNES(DriverBase):
                     scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
                     if scaled:
                         if DEBUG:
-                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
-                                          (metadata.cover,width,height,nwidth,nheight))
+                            self.log.info("   cover scaled from %sx%s to %sx%s" %
+                                          (width,height,nwidth,nheight))
                         img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
                         cd = cStringIO.StringIO()
                         img.convert('RGB').save(cd, 'JPEG')
@@ -1337,9 +1374,11 @@ class ITUNES(DriverBase):
                     return thumb
 
                 if isosx:
-                    # The following commands generate an error, but the artwork does in fact
-                    # get sent to the device.  Seems like a bug in Apple's automation interface?
-                    # Could also be a problem with the integrity of the cover data?
+                    '''
+                    The following commands generate an error, but the artwork does in fact
+                    get sent to the device.  Seems like a bug in Apple's automation interface?
+                    Could also be a problem with the integrity of the cover data?
+                    '''
                     if lb_added:
                         try:
                             lb_added.artworks[1].data_.set(cover_data)
@@ -1362,9 +1401,8 @@ class ITUNES(DriverBase):
                             #ipython(user_ns=locals())
                             pass
 
-
                 elif iswindows:
-                    # Write the data to a real file for Windows iTunes
+                    ''' Write the data to a real file for Windows iTunes '''
                     tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
                     with open(tc,'wb') as tmp_cover:
                         tmp_cover.write(cover_data)
@@ -1423,7 +1461,8 @@ class ITUNES(DriverBase):
 
         this_book = Book(metadata.title, authors_to_string(metadata.authors))
         this_book.datetime = time.gmtime()
-        this_book.db_id = None
+        #this_book.cid = metadata.id
+        this_book.cid = None
         this_book.device_collections = []
         this_book.format = format
         this_book.library_id = lb_added     # ??? GR
@@ -1431,7 +1470,6 @@ class ITUNES(DriverBase):
         this_book.thumbnail = thumb
         this_book.iTunes_id = lb_added      # ??? GR
         this_book.uuid = metadata.uuid
-
         if isosx:
             if lb_added:
                 this_book.size = self._get_device_book_size(fpath, lb_added.size())
@@ -1462,24 +1500,6 @@ class ITUNES(DriverBase):
 
         return this_book
 
-    def _delete_iTunesMetadata_plist(self,fpath):
-        '''
-        Delete the plist file from the file to force recache
-        '''
-        zf = ZipFile(fpath,'a')
-        fnames = zf.namelist()
-        pl_name = 'iTunesMetadata.plist'
-        try:
-            plist = [x for x in fnames if pl_name in x][0]
-        except:
-            plist = None
-        if plist:
-            if DEBUG:
-                self.log.info("  _delete_iTunesMetadata_plist():")
-                self.log.info("    deleting '%s'\n   from '%s'" % (pl_name,fpath))
-                zf.delete(pl_name)
-        zf.close()
-
     def _discover_manual_sync_mode(self, wait=0):
         '''
         Assumes pythoncom for windows
@@ -1664,18 +1684,6 @@ class ITUNES(DriverBase):
         zf.close()
         return (title, author, timestamp)
 
-    def _dump_files(self, files, header=None,indent=0):
-        if header:
-            msg = '\n%sfiles passed to %s:' % (' '*indent,header)
-            self.log.info(msg)
-            self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
-        for file in files:
-            if getattr(file, 'orig_file_path', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.orig_file_path))
-            elif getattr(file, 'name', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.name))
-        self.log.info()
-
     def _dump_hex(self, src, length=16):
         '''
         '''
@@ -1699,7 +1707,7 @@ class ITUNES(DriverBase):
         self.log.info()
 
     def _dump_update_list(self,header=None,indent=0):
-        if header:
+        if header and self.update_list:
             msg = '\n%sself.update_list %s' % (' '*indent,header)
             self.log.info(msg)
             self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
@@ -1718,7 +1726,6 @@ class ITUNES(DriverBase):
                  (' '*indent,
                   ub['title'],
                   ub['author']))
-        self.log.info()
 
     def _find_device_book(self, search):
         '''
@@ -2117,35 +2124,6 @@ class ITUNES(DriverBase):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, format, update_md=False):
-        '''
-        If the database copy will be deleted after upload, we have to
-        use file (the PersistentTemporaryFile), which will be around until
-        calibre exits.
-        If we're using the database copy, delete the plist
-        '''
-        if DEBUG:
-            self.log.info(" ITUNES._get_fpath()")
-
-        fpath = file
-        if not getattr(fpath, 'deleted_after_upload', False):
-            if getattr(file, 'orig_file_path', None) is not None:
-                # Database copy
-                fpath = file.orig_file_path
-                self._delete_iTunesMetadata_plist(fpath)
-            elif getattr(file, 'name', None) is not None:
-                # PTF
-                fpath = file.name
-        else:
-            # Recipe - PTF
-            if DEBUG:
-                self.log.info("   file will be deleted after upload")
-
-        if format == 'epub' and update_md:
-            self._update_epub_metadata(fpath, metadata)
-
-        return fpath
-
     def _get_library_books(self):
         '''
         Populate a dict of paths from iTunes Library|Books
@@ -2349,6 +2327,7 @@ class ITUNES(DriverBase):
                 self.iTunes = appscript.app('iTunes')
                 self.initial_status = 'already running'
 
+            '''
             # Read the current storage path for iTunes media
             cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
             proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
@@ -2359,12 +2338,13 @@ class ITUNES(DriverBase):
             else:
                 self.log.error("  could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
                 self.log.error("  media_dir: %s" % media_dir)
+            '''
+
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [OSX %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
         if iswindows:
@@ -2404,6 +2384,7 @@ class ITUNES(DriverBase):
                              ' iTunes automation interface non-responsive, ' +
                              'recommend reinstalling iTunes')
 
+            '''
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2422,13 +2403,13 @@ class ITUNES(DriverBase):
                     self.log.error("  '%s' not found" % media_dir)
                 else:
                     self.log.error("  no media dir found: string: %s" % string)
+            '''
 
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [Windows %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
     def _purge_orphans(self,library_books, cached_books):
@@ -2478,13 +2459,14 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                    self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_device(self.cached_books[book])
+
                     if DEBUG:
                         self.log.info( "  deleting device book '%s'" % (metadata.title))
-                    if not getattr(file, 'deleted_after_upload', False):
-                        self._remove_from_iTunes(self.cached_books[book])
-                        if DEBUG:
-                            self.log.info("  deleting library book '%s'" % metadata.title)
+                    self._remove_from_device(self.cached_books[book])
+
+                    if DEBUG:
+                        self.log.info("  deleting library book '%s'" % metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2497,9 +2479,9 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                     self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
                         self.log.info( "  deleting library book '%s'" %  metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2509,7 +2491,8 @@ class ITUNES(DriverBase):
         '''
         Windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._remove_from_device()")
+        if DEBUG:
+            self.log.info(" ITUNES._remove_from_device()")
         if isosx:
             if DEBUG:
                 self.log.info("  deleting '%s' from iDevice" % cached_book['title'])
@@ -2530,96 +2513,105 @@ class ITUNES(DriverBase):
 
     def _remove_from_iTunes(self, cached_book):
         '''
-        iTunes does not delete books from storage when removing from database
-        We only want to delete stored copies if the file is stored in iTunes
-        We don't want to delete files stored outside of iTunes.
-        Also confirm that storage_path does not point into calibre's storage.
+        iTunes does not delete books from storage when removing from database via automation
         '''
         if DEBUG:
             self.log.info(" ITUNES._remove_from_iTunes():")
 
         if isosx:
+            ''' Manually remove the book from iTunes storage '''
             try:
-                storage_path = os.path.split(cached_book['lib_book'].location().path)
-                if cached_book['lib_book'].location().path.startswith(self.iTunes_media) and \
-                   not storage_path[0].startswith(prefs['library_path']):
-                    title_storage_path = storage_path[0]
-                    if DEBUG:
-                        self.log.info("  removing title_storage_path: %s" % title_storage_path)
-                    try:
-                        shutil.rmtree(title_storage_path)
-                    except:
-                        self.log.info("  '%s' not empty" % title_storage_path)
-
-                    # Clean up title/author directories
-                    author_storage_path = os.path.split(title_storage_path)[0]
-                    self.log.info("  author_storage_path: %s" % author_storage_path)
-                    author_files = os.listdir(author_storage_path)
-                    if '.DS_Store' in author_files:
-                        author_files.pop(author_files.index('.DS_Store'))
-                    if not author_files:
-                        shutil.rmtree(author_storage_path)
-                        if DEBUG:
-                            self.log.info("  removing empty author_storage_path")
-                    else:
-                        if DEBUG:
-                            self.log.info("  author_storage_path not empty (%d objects):" % len(author_files))
-                            self.log.info("  %s" % '\n'.join(author_files))
+                fp = cached_book['lib_book'].location().path
+                if DEBUG:
+                    self.log.info("  processing %s" % fp)
+                if fp.startswith(prefs['library_path']):
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
                 else:
-                    self.log.info("  '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
+                        try:
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
+                        except:
+                            author_files = os.listdir(author_storage_path)
+                            if '.DS_Store' in author_files:
+                                author_files.pop(author_files.index('.DS_Store'))
+                            if not author_files:
+                                os.rmdir(author_storage_path)
+                                if DEBUG:
+                                    self.log.info("   removing empty author directory")
+                            '''
+                            else:
+                                if DEBUG:
+                                    self.log.info("   author_storage_path not empty:")
+                                    self.log.info("   %s" % '\n'.join(author_files))
+                            '''
+                    else:
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
 
             except:
                 # We get here if there was an error with .location().path
                 if DEBUG:
-                    self.log.info("   '%s' not in iTunes storage" % cached_book['title'])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
 
+            # Delete the book from the iTunes database
             try:
                 self.iTunes.delete(cached_book['lib_book'])
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
         elif iswindows:
             '''
             Assume we're wrapped in a pythoncom
             Windows stores the book under a common author directory, so we just delete the .epub
             '''
+            fp = None
             try:
                 book = cached_book['lib_book']
-                path = book.Location
+                fp = book.Location
             except:
                 book = self._find_library_book(cached_book)
                 if book:
-                    path = book.Location
+                    fp = book.Location
 
             if book:
-                if self.iTunes_media and path.startswith(self.iTunes_media) and \
-                   not path.startswith(prefs['library_path']):
-                    storage_path = os.path.split(path)
-                    if DEBUG:
-                        self.log.info("   removing '%s' at %s" %
-                            (cached_book['title'], path))
-                    try:
-                        os.remove(path)
-                    except:
-                        self.log.warning("   '%s' not in iTunes storage" % path)
-                    try:
-                        os.rmdir(storage_path[0])
-                        self.log.info("   removed folder '%s'" % storage_path[0])
-                    except:
-                        self.log.info("   folder '%s' not found or not empty" % storage_path[0])
-
-                    # Delete from iTunes database
+                if DEBUG:
+                    self.log.info("  processing %s" % fp)
+                if fp.startswith(prefs['library_path']):
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
                 else:
-                    self.log.info("   '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
+                        try:
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
+                        except:
+                            pass
+                    else:
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
             else:
                 if DEBUG:
-                    self.log.info("   '%s' not found in iTunes" % cached_book['title'])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
+
+            # Delete the book from the iTunes database
             try:
                 book.Delete()
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
     def title_sorter(self, title):
         return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
@@ -2627,7 +2619,8 @@ class ITUNES(DriverBase):
     def _update_epub_metadata(self, fpath, metadata):
         '''
         '''
-        self.log.info(" ITUNES._update_epub_metadata()")
+        if DEBUG:
+            self.log.info(" ITUNES._update_epub_metadata()")
 
         # Fetch plugboard updates
         metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub')
@@ -2798,7 +2791,7 @@ class ITUNES(DriverBase):
             if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]:
                 if DEBUG:
                     self.log.info(" ITUNES._update_iTunes_metadata()")
-                    self.log.info("   using Series name as Genre")
+                    self.log.info("   using Series name '%s' as Genre" % metadata_x.series)
 
                 # Format the index as a sort key
                 index = metadata_x.series_index
@@ -2978,8 +2971,8 @@ class ITUNES(DriverBase):
             newmi = book.deepcopy_metadata()
             newmi.template_to_attribute(book, pb)
             if pb is not None and DEBUG:
-                self.log.info(" transforming %s using %s:" % (format, pb))
-                self.log.info("       title: %s %s" % (book.title, ">>> %s" %
+                #self.log.info(" transforming %s using %s:" % (format, pb))
+                self.log.info("       title: '%s' %s" % (book.title, ">>> '%s'" %
                                            newmi.title if book.title != newmi.title else ''))
                 self.log.info("  title_sort: %s %s" % (book.title_sort, ">>> %s" %
                                            newmi.title_sort if book.title_sort != newmi.title_sort else ''))
@@ -2994,7 +2987,8 @@ class ITUNES(DriverBase):
                 self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
                                            newmi.tags if book.tags != newmi.tags else ''))
             else:
-                self.log("  matching plugboard not found")
+                if DEBUG:
+                    self.log("  matching plugboard not found")
 
         else:
             newmi = book
@@ -3083,12 +3077,12 @@ class ITUNES_ASYNC(ITUNES):
                         this_book.datetime = parse_date(str(library_books[book].date_added())).timetuple()
                     except:
                         this_book.datetime = time.gmtime()
-                    this_book.db_id = None
                     this_book.device_collections = []
                     #this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                     this_book.library_id = library_books[book]
                     this_book.size = library_books[book].size()
                     this_book.uuid = library_books[book].composer()
+                    this_book.cid = None
                     # Hack to discover if we're running in GUI environment
                     if self.report_progress is not None:
                         this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])
@@ -3124,11 +3118,11 @@ class ITUNES_ASYNC(ITUNES):
                             this_book.datetime = parse_date(str(library_books[book].DateAdded)).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[book]
                         this_book.size = library_books[book].Size
                         this_book.uuid = library_books[book].Composer
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])

src/calibre/devices/kobo/driver.py

@@ -125,9 +125,9 @@ class KOBO(USBMS):
                 # this shows an expired Collection so the user can decide to delete the book
                 if expired == 3:
                     playlist_map[lpath].append('Expired')
-                # Favourites are supported on the touch but the data field is there on most earlier models
+                # A SHORTLIST is supported on the touch but the data field is there on most earlier models
                 if favouritesindex == 1:
-                    playlist_map[lpath].append('Favourite')
+                    playlist_map[lpath].append('Shortlist')
 
                 path = self.normalize_path(path)
                 # print "Normalized FileName: " + path
@@ -557,6 +557,7 @@ class KOBO(USBMS):
         if collections:
             # Process any collections that exist
             for category, books in collections.items():
+                # debug_print (category)
                 if category == 'Im_Reading':
                     # Reset Im_Reading list in the database
                     if oncard == 'carda':
@@ -575,7 +576,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                        debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Im_Reading']
+                        if 'Im_Reading' not in book.device_collections:
+                            book.device_collections.append('Im_Reading') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -618,7 +620,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                       debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Read']
+                        if 'Read' not in book.device_collections:
+                            book.device_collections.append('Read') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -654,7 +657,8 @@ class KOBO(USBMS):
 
                     for book in books:
 #                       debug_print('Title:', book.title, 'lpath:', book.path)
-                        book.device_collections = ['Closed']
+                        if 'Closed' not in book.device_collections:
+                            book.device_collections.append('Closed') 
 
                         extension =  os.path.splitext(book.path)[1]
                         ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
@@ -672,6 +676,44 @@ class KOBO(USBMS):
                         else:
                             connection.commit()
 #                            debug_print('Database: Commit set ReadStatus as Closed')
+                if category == 'Shortlist':
+                    # Reset FavouritesIndex list in the database
+                    if oncard == 'carda':
+                        query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID like \'file:///mnt/sd/%\''
+                    elif oncard != 'carda' and oncard != 'cardb':
+                        query= 'update content set FavouritesIndex=-1 where BookID is Null and ContentID not like \'file:///mnt/sd/%\''
+
+                    try:
+                        cursor.execute (query)
+                    except:
+                        debug_print('Database Exception:  Unable to reset Shortlist list')
+                        raise
+                    else:
+#                       debug_print('Commit: Reset Shortlist list')
+                        connection.commit()
+
+                    for book in books:
+#                        debug_print('Title:', book.title, 'lpath:', book.path)
+                        if 'Shortlist' not in book.device_collections:
+                            book.device_collections.append('Shortlist') 
+                        # debug_print ("Shortlist found for: ", book.title) 
+                        extension =  os.path.splitext(book.path)[1]
+                        ContentType = self.get_content_type_from_extension(extension) if extension != '' else self.get_content_type_from_path(book.path)
+
+                        ContentID = self.contentid_from_path(book.path, ContentType)
+#                        datelastread = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime())
+
+                        t = (ContentID,)
+
+                        try:
+                            cursor.execute('update content set FavouritesIndex=1 where BookID is Null and ContentID = ?', t)
+                        except:
+                            debug_print('Database Exception:  Unable set book as Shortlist')
+                            raise
+                        else:
+                            connection.commit()
+#                            debug_print('Database: Commit set Shortlist as Shortlist')
+
         else: # No collections
             # Since no collections exist the ReadStatus needs to be reset to 0 (Unread)
             print "Reseting ReadStatus to 0"

src/calibre/devices/teclast/driver.py

@@ -19,8 +19,9 @@ class TECLAST_K3(USBMS):
     PRODUCT_ID  = [0x3203]
     BCD         = [0x0000, 0x0100]
 
-    VENDOR_NAME      = 'TECLAST'
-    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5']
+    VENDOR_NAME      = ['TECLAST', 'IMAGIN']
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['DIGITAL_PLAYER', 'TL-K5',
+            'EREADER']
 
     MAIN_MEMORY_VOLUME_LABEL  = 'K3 Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'K3 Storage Card'

src/calibre/ebooks/metadata/fb2.py

@@ -1,96 +1,235 @@
 #!/usr/bin/env python
 from __future__ import with_statement
 __license__   = 'GPL v3'
-__copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
-
+__copyright__ = '2011, Roman Mukhin <ramses_ru at hotmail.com>, '\
+                '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 '''Read meta information from fb2 files'''
 
 import os
+import datetime
+from functools import partial
 from base64 import b64decode
 from lxml import etree
-from calibre.ebooks.metadata import MetaInformation
+from calibre.utils.date import parse_date
+from calibre import guess_all_extensions, prints, force_unicode
+from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.chardet import xml_to_unicode
-from calibre import guess_all_extensions
 
-XLINK_NS     = 'http://www.w3.org/1999/xlink'
-def XLINK(name):
-    return '{%s}%s' % (XLINK_NS, name)
 
+NAMESPACES = {
+    'fb2'   :   'http://www.gribuser.ru/xml/fictionbook/2.0',
+    'xlink' :   'http://www.w3.org/1999/xlink'  }
+
+XPath = partial(etree.XPath, namespaces=NAMESPACES)
+tostring = partial(etree.tostring, method='text', encoding=unicode)
 
 def get_metadata(stream):
-    """ Return metadata as a L{MetaInfo} object """
-    XPath = lambda x : etree.XPath(x,
-            namespaces={'fb2':'http://www.gribuser.ru/xml/fictionbook/2.0',
-                'xlink':XLINK_NS})
-    tostring = lambda x : etree.tostring(x, method='text',
-            encoding=unicode).strip()
+    """ Return fb2 metadata as a L{MetaInformation} object """
+
+    root = _get_fbroot(stream)
+
+    book_title = _parse_book_title(root)
+    authors = _parse_authors(root)
+
+    # fallback for book_title
+    if book_title:
+        book_title = unicode(book_title)
+    else:
+        book_title = force_unicode(os.path.splitext(
+            os.path.basename(getattr(stream, 'name',
+                _('Unknown'))))[0])
+    mi = MetaInformation(book_title, authors)
+
+    try:
+        _parse_cover(root, mi)
+    except:
+        pass
+    try:
+        _parse_comments(root, mi)
+    except:
+        pass
+    try:
+        _parse_tags(root, mi)
+    except:
+        pass
+    try:
+        _parse_series(root, mi)
+    except:
+        pass
+    try:
+        _parse_isbn(root, mi)
+    except:
+        pass
+    try:
+        _parse_publisher(root, mi)
+    except:
+        pass
+    try:
+        _parse_pubdate(root, mi)
+    except:
+        pass
+    try:
+        _parse_timestamp(root, mi)
+    except:
+        pass
+
+    try:
+        _parse_language(root, mi)
+    except:
+        pass
+    #_parse_uuid(root, mi)
+
+    #if DEBUG:
+    #   prints(mi)
+    return mi
+
+def _parse_authors(root):
+    authors = []
+    # pick up authors but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>, <document-info>
+    for author_sec in ['title-info', 'src-title-info', 'document-info']:
+        for au in XPath('//fb2:%s/fb2:author'%author_sec)(root):
+            author = _parse_author(au)
+            if author:
+                authors.append(author)
+        if author:
+            break
+
+    # if no author so far
+    if not authors:
+        authors.append(_('Unknown'))
+
+    return authors
+
+def _parse_author(elm_author):
+    """ Returns a list of display author and sortable author"""
+
+    xp_templ = 'normalize-space(fb2:%s/text())'
+
+    author = XPath(xp_templ % 'first-name')(elm_author)
+    lname = XPath(xp_templ % 'last-name')(elm_author)
+    mname = XPath(xp_templ % 'middle-name')(elm_author)
+
+    if mname:
+        author = (author + ' ' + mname).strip()
+    if lname:
+        author = (author + ' ' + lname).strip()
+
+    # fallback to nickname
+    if not author:
+        nname = XPath(xp_templ % 'nickname')(elm_author)
+        if nname:
+            author = nname
+
+    return author
+
+
+def _parse_book_title(root):
+    # <title-info> has a priority.   (actually <title-info>  is mandatory)
+    # other are backup solution (sequence is important. other then in fb2-doc)
+    xp_ti = '//fb2:title-info/fb2:book-title/text()'
+    xp_pi = '//fb2:publish-info/fb2:book-title/text()'
+    xp_si = '//fb2:src-title-info/fb2:book-title/text()'
+    book_title = XPath('normalize-space(%s|%s|%s)' % (xp_ti, xp_pi, xp_si))(root)
+
+    return book_title
+
+def _parse_cover(root, mi):
+    # pickup from <title-info>, if not exists it fallbacks to <src-title-info>
+    imgid = XPath('substring-after(string(//fb2:coverpage/fb2:image/@xlink:href), "#")')(root)
+    if imgid:
+        try:
+            _parse_cover_data(root, imgid, mi)
+        except:
+            pass
+
+def _parse_cover_data(root, imgid, mi):
+    elm_binary = XPath('//fb2:binary[@id="%s"]'%imgid)(root)
+    if elm_binary:
+        mimetype = elm_binary[0].get('content-type', 'image/jpeg')
+        mime_extensions = guess_all_extensions(mimetype)
+        if mime_extensions:
+            pic_data = elm_binary[0].text
+            if pic_data:
+                mi.cover_data = (mime_extensions[0][1:], b64decode(pic_data))
+        else:
+            prints("WARNING: Unsupported coverpage mime-type '%s' (id=#%s)" % (mimetype, imgid) )
+
+def _parse_tags(root, mi):
+    # pick up genre but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>
+    for genre_sec in ['title-info', 'src-title-info']:
+        # -- i18n Translations-- ?
+        tags = XPath('//fb2:%s/fb2:genre/text()' % genre_sec)(root)
+        if tags:
+            mi.tags = list(map(unicode, tags))
+            break
+
+def _parse_series(root, mi):
+    #calibri supports only 1 series: use the 1-st one
+    # pick up sequence but only from 1 secrion in prefered order
+    # except <src-title-info>
+    xp_ti = '//fb2:title-info/fb2:sequence[1]'
+    xp_pi = '//fb2:publish-info/fb2:sequence[1]'
+
+    elms_sequence = XPath('%s|%s' % (xp_ti, xp_pi))(root)
+    if elms_sequence:
+        mi.series = elms_sequence[0].get('name', None)
+        if mi.series:
+            mi.series_index = elms_sequence[0].get('number', None)
+
+def _parse_isbn(root, mi):
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    isbn = XPath('normalize-space(//fb2:publish-info/fb2:isbn/text())')(root)
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    if ',' in isbn:
+        isbn = isbn[:isbn.index(',')]
+    if check_isbn(isbn):
+        mi.isbn = isbn
+
+def _parse_comments(root, mi):
+    # pick up annotation but only from 1 secrion <title-info>;  fallback: <src-title-info>
+    for annotation_sec in ['title-info', 'src-title-info']:
+        elms_annotation = XPath('//fb2:%s/fb2:annotation' % annotation_sec)(root)
+        if elms_annotation:
+            mi.comments = tostring(elms_annotation[0])
+            # TODO: tags i18n, xslt?
+            break
+
+def _parse_publisher(root, mi):
+    publisher = XPath('string(//fb2:publish-info/fb2:publisher/text())')(root)
+    if publisher:
+        mi.publisher = publisher
+
+def _parse_pubdate(root, mi):
+    year = XPath('number(//fb2:publish-info/fb2:year/text())')(root)
+    if float.is_integer(year):
+        # only year is available, so use 1-st of Jan
+        mi.pubdate = datetime.date(int(year), 1, 1)
+
+def _parse_timestamp(root, mi):
+    #<date value="1996-12-03">03.12.1996</date>
+    xp ='//fb2:document-info/fb2:date/@value|'\
+        '//fb2:document-info/fb2:date/text()'
+    docdate = XPath('string(%s)' % xp)(root)
+    if docdate:
+        mi.timestamp = parse_date(docdate)
+
+def _parse_language(root, mi):
+    language = XPath('string(//fb2:title-info/fb2:lang/text())')(root)
+    if language:
+        mi.language = language
+        mi.languages = [ language ]
+
+def _parse_uuid(root, mi):
+    uuid = XPath('normalize-space(//document-info/fb2:id/text())')(root)
+    if uuid:
+        mi.uuid = uuid
+
+def _get_fbroot(stream):
     parser = etree.XMLParser(recover=True, no_network=True)
     raw = stream.read()
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
-            assume_utf8=True)[0]
+    raw = xml_to_unicode(raw, strip_encoding_pats=True)[0]
     root = etree.fromstring(raw, parser=parser)
-    authors, author_sort = [], None
-    for au in XPath('//fb2:author')(root):
-        fname = lname = author = None
-        fe = XPath('descendant::fb2:first-name')(au)
-        if fe:
-            fname = tostring(fe[0])
-            author = fname
-        le = XPath('descendant::fb2:last-name')(au)
-        if le:
-            lname = tostring(le[0])
-            if author:
-                author += ' '+lname
-            else:
-                author = lname
-        if author:
-            authors.append(author)
-        if len(authors) == 1 and author is not None:
-            if lname:
-                author_sort = lname
-            if fname:
-                if author_sort: author_sort += ', '+fname
-                else: author_sort = fname
-    title = os.path.splitext(os.path.basename(getattr(stream, 'name',
-        _('Unknown'))))[0]
-    for x in XPath('//fb2:book-title')(root):
-        title = tostring(x)
-        break
-    comments = ''
-    for x in XPath('//fb2:annotation')(root):
-        comments += tostring(x)
-    if not comments:
-        comments = None
-    tags = list(map(tostring, XPath('//fb2:genre')(root)))
+    return root
 
-    cp = XPath('//fb2:coverpage')(root)
-    cdata = None
-    if cp:
-        cimage = XPath('descendant::fb2:image[@xlink:href]')(cp[0])
-        if cimage:
-            id = cimage[0].get(XLINK('href')).replace('#', '')
-            binary = XPath('//fb2:binary[@id="%s"]'%id)(root)
-            if binary:
-                mt = binary[0].get('content-type', 'image/jpeg')
-                exts = guess_all_extensions(mt)
-                if not exts:
-                    exts = ['.jpg']
-                cdata = (exts[0][1:], b64decode(tostring(binary[0])))
-
-    series = None
-    series_index = 1.0
-    for x in XPath('//fb2:sequence')(root):
-        series = x.get('name', None)
-        if series is not None:
-            series_index = x.get('number', 1.0)
-            break
-    mi = MetaInformation(title, authors)
-    mi.comments = comments
-    mi.author_sort = author_sort
-    if tags:
-        mi.tags = tags
-    mi.series = series
-    mi.series_index = series_index
-    if cdata:
-        mi.cover_data = cdata
-    return mi

src/calibre/gui2/__init__.py

@@ -248,10 +248,11 @@ def error_dialog(parent, title, msg, det_msg='', show=False,
         return d.exec_()
     return d
 
-def question_dialog(parent, title, msg, det_msg='', show_copy_button=False):
+def question_dialog(parent, title, msg, det_msg='', show_copy_button=False,
+        default_yes=True):
     from calibre.gui2.dialogs.message_box import MessageBox
     d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent,
-                    show_copy_button=show_copy_button)
+                    show_copy_button=show_copy_button, default_yes=default_yes)
     return d.exec_() == d.Accepted
 
 def info_dialog(parent, title, msg, det_msg='', show=False,

src/calibre/gui2/actions/choose_library.py

@@ -252,11 +252,12 @@ class ChooseLibraryAction(InterfaceAction):
 
     def delete_requested(self, name, location):
         loc = location.replace('/', os.sep)
-        if not question_dialog(self.gui, _('Are you sure?'), '<p>'+
+        if not question_dialog(self.gui, _('Are you sure?'),
+                _('<h1 style="color:red">WARNING</h1>')+
                 _('<b style="color: red">All files</b> (not just ebooks) '
                     'from <br><br><b>%s</b><br><br> will be '
                 '<b>permanently deleted</b>. Are you sure?') % loc,
-                show_copy_button=False):
+                show_copy_button=False, default_yes=False):
             return
         exists = self.gui.library_view.model().db.exists_at(loc)
         if exists:

src/calibre/gui2/add.py

@@ -451,7 +451,8 @@ class Saver(QObject): # {{{
         self.callback_called = False
         self.rq = Queue()
         self.ids = [x for x in map(db.id, [r.row() for r in rows]) if x is not None]
-        self.pd.set_max(len(self.ids))
+        self.pd_max = len(self.ids)
+        self.pd.set_max(0)
         self.pd.value = 0
         self.failures = set([])
 
@@ -510,6 +511,8 @@ class Saver(QObject): # {{{
             id, title, ok, tb = self.rq.get_nowait()
         except Empty:
             return
+        if self.pd.max != self.pd_max:
+            self.pd.max = self.pd_max
         self.pd.value += 1
         self.ids.remove(id)
         if not isinstance(title, unicode):

src/calibre/gui2/custom_column_widgets.py

@@ -25,7 +25,7 @@ class Base(object):
     def __init__(self, db, col_id, parent=None):
         self.db, self.col_id = db, col_id
         self.col_metadata = db.custom_column_num_map[col_id]
-        self.initial_val = None
+        self.initial_val = self.widgets = None
         self.setup_ui(parent)
 
     def initialize(self, book_id):
@@ -54,6 +54,9 @@ class Base(object):
     def normalize_ui_val(self, val):
         return val
 
+    def break_cycles(self):
+        self.db = self.widgets = self.initial_val = None
+
 class Bool(Base):
 
     def setup_ui(self, parent):

src/calibre/gui2/dialogs/choose_library.ui

@@ -41,7 +41,7 @@
    <item row="4" column="0" colspan="4">
     <widget class="QRadioButton" name="existing_library">
      <property name="text">
-      <string>Use &amp;existing library at the new location</string>
+      <string>Use the previously &amp;existing library at the new location</string>
      </property>
      <property name="checked">
       <bool>true</bool>

src/calibre/gui2/dialogs/message_box.py

@@ -23,7 +23,7 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
                  det_msg='',
                  q_icon=None,
                  show_copy_button=True,
-                 parent=None):
+                 parent=None, default_yes=True):
         QDialog.__init__(self, parent)
         if q_icon is None:
             icon = {
@@ -65,7 +65,9 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         self.is_question = type_ == self.QUESTION
         if self.is_question:
             self.bb.setStandardButtons(self.bb.Yes|self.bb.No)
-            self.bb.button(self.bb.Yes).setDefault(True)
+            self.bb.button(self.bb.Yes if default_yes else self.bb.No
+                    ).setDefault(True)
+            self.default_yes = default_yes
         else:
             self.bb.button(self.bb.Ok).setDefault(True)
 
@@ -101,7 +103,8 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         ret = QDialog.showEvent(self, ev)
         if self.is_question:
             try:
-                self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason)
+                self.bb.button(self.bb.Yes if self.default_yes else self.bb.No
+                        ).setFocus(Qt.OtherFocusReason)
             except:
                 pass# Buttons were changed
         else:

src/calibre/gui2/dialogs/progress.py

@@ -53,6 +53,13 @@ class ProgressDialog(QDialog, Ui_Dialog):
     def set_max(self, max):
         self.bar.setMaximum(max)
 
+    @dynamic_property
+    def max(self):
+        def fget(self): return self.bar.maximum()
+        def fset(self, val): self.bar.setMaximum(val)
+        return property(fget=fget, fset=fset)
+
+
     def _canceled(self, *args):
         self.canceled = True
         self.button_box.setDisabled(True)

src/calibre/gui2/dialogs/quickview.py

@@ -5,11 +5,13 @@ __docformat__ = 'restructuredtext en'
 
 
 from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem,
-                      QListWidgetItem, QByteArray, QModelIndex, QCoreApplication)
+                      QListWidgetItem, QByteArray, QCoreApplication,
+                      QApplication)
 
+from calibre.customize.ui import find_plugin
+from calibre.gui2 import gprefs
 from calibre.gui2.dialogs.quickview_ui import Ui_Quickview
 from calibre.utils.icu import sort_key
-from calibre.gui2 import gprefs
 
 class TableItem(QTableWidgetItem):
     '''
@@ -55,8 +57,9 @@ class Quickview(QDialog, Ui_Quickview):
         self.is_closed = False
         self.current_book_id = None
         self.current_key = None
-        self.use_current_key_for_next_refresh = False
         self.last_search = None
+        self.current_column = None
+        self.current_item = None
 
         self.items.setSelectionMode(QAbstractItemView.SingleSelection)
         self.items.currentTextChanged.connect(self.item_selected)
@@ -87,16 +90,24 @@ class Quickview(QDialog, Ui_Quickview):
         # Add the data
         self.refresh(row)
 
-        self.view.selectionModel().currentChanged[QModelIndex,QModelIndex].connect(self.slave)
+        self.view.clicked.connect(self.slave)
         QCoreApplication.instance().aboutToQuit.connect(self.save_state)
         self.search_button.clicked.connect(self.do_search)
+        view.model().new_bookdisplay_data.connect(self.book_was_changed)
 
     # search button
     def do_search(self):
         if self.last_search is not None:
-            self.use_current_key_for_next_refresh = True
             self.gui.search.set_search_string(self.last_search)
 
+    # Called when book information is changed in the library view. Make that
+    # book current. This means that prev and next in edit metadata will move
+    # the current book.
+    def book_was_changed(self, mi):
+        if self.is_closed or self.current_column is None:
+            return
+        self.refresh(self.view.model().index(self.db.row(mi.id), self.current_column))
+
     # clicks on the items listWidget
     def item_selected(self, txt):
         self.fill_in_books_box(unicode(txt))
@@ -104,22 +115,15 @@ class Quickview(QDialog, Ui_Quickview):
     # Given a cell in the library view, display the information
     def refresh(self, idx):
         bv_row = idx.row()
-        key = self.view.model().column_map[idx.column()]
-
+        self.current_column = idx.column()
+        key = self.view.model().column_map[self.current_column]
         book_id = self.view.model().id(bv_row)
 
-        # Double-clicking on a book to show it in the library view will result
-        # in a signal emitted for column 1 of the book row. Use the original
-        # column for this signal.
-        if self.use_current_key_for_next_refresh:
+        # Only show items for categories
+        if not self.db.field_metadata[key]['is_category']:
+            if self.current_key is None:
+                return
             key = self.current_key
-            self.use_current_key_for_next_refresh = False
-        else:
-            # Only show items for categories
-            if not self.db.field_metadata[key]['is_category']:
-                if self.current_key is None:
-                    return
-                key = self.current_key
         self.items_label.setText('{0} ({1})'.format(
                                     self.db.field_metadata[key]['name'], key))
 
@@ -147,6 +151,7 @@ class Quickview(QDialog, Ui_Quickview):
         self.items.blockSignals(False)
 
     def fill_in_books_box(self, selected_item):
+        self.current_item = selected_item
         # Do a bit of fix-up on the items so that the search works.
         if selected_item.startswith('.'):
             sv = '.' + selected_item
@@ -162,19 +167,26 @@ class Quickview(QDialog, Ui_Quickview):
 
         select_item = None
         self.books_table.setSortingEnabled(False)
+        tt = ('<p>' +
+            _('Double-click on a book to change the selection in the library view. '
+               'Shift- or control-double-click to edit the metadata of a book')
+              + '</p>')
         for row, b in enumerate(books):
             mi = self.db.get_metadata(b, index_is_id=True, get_user_categories=False)
             a = TableItem(mi.title, mi.title_sort)
             a.setData(Qt.UserRole, b)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 0, a)
             if b == self.current_book_id:
                 select_item = a
             a = TableItem(' & '.join(mi.authors), mi.author_sort)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 1, a)
             series = mi.format_field('series')[1]
             if series is None:
                 series = ''
             a = TableItem(series, series)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 2, a)
             self.books_table.setRowHeight(row, self.books_table_row_height)
 
@@ -201,11 +213,16 @@ class Quickview(QDialog, Ui_Quickview):
         self.save_state()
 
     def book_doubleclicked(self, row, column):
-        self.use_current_key_for_next_refresh = True
-        self.view.select_rows([self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]])
+        book_id = self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]
+        self.view.select_rows([book_id])
+        modifiers = int(QApplication.keyboardModifiers())
+        if modifiers in (Qt.CTRL, Qt.SHIFT):
+            em = find_plugin('Edit Metadata')
+            if em is not None:
+                em.actual_plugin_.edit_metadata(None)
 
     # called when a book is clicked on the library view
-    def slave(self, current, previous):
+    def slave(self, current):
         if self.is_closed:
             return
         self.refresh(current)

src/calibre/gui2/library/views.py

@@ -591,8 +591,10 @@ class BooksView(QTableView): # {{{
         fmt = prefs['output_format']
 
         def url_for_id(i):
-            ans = db.format(i, fmt, index_is_id=True, as_path=True,
-                    preserve_filename=True)
+            try:
+                ans = db.format_path(i, fmt, index_is_id=True)
+            except:
+                ans = None
             if ans is None:
                 fmts = db.formats(i, index_is_id=True)
                 if fmts:
@@ -600,13 +602,15 @@ class BooksView(QTableView): # {{{
                 else:
                     fmts = []
                 for f in fmts:
-                    ans = db.format(i, f, index_is_id=True, as_path=True,
-                            preserve_filename=True)
+                    try:
+                        ans = db.format_path(i, f, index_is_id=True)
+                    except:
+                        ans = None
             if ans is None:
                 ans = db.abspath(i, index_is_id=True)
             return QUrl.fromLocalFile(ans)
 
-        md.setUrls([url_for_id(i) for i in selected[:25]])
+        md.setUrls([url_for_id(i) for i in selected])
         drag = QDrag(self)
         col = self.selectionModel().currentIndex().column()
         md.column_name = self.column_map[col]

src/calibre/gui2/metadata/basic_widgets.py

@@ -21,9 +21,10 @@ from calibre.utils.config import tweaks, prefs
 from calibre.ebooks.metadata import (title_sort, authors_to_string,
         string_to_authors, check_isbn, authors_to_sort_string)
 from calibre.ebooks.metadata.meta import get_metadata
-from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE, UNDEFINED_DATE,
+from calibre.gui2 import (file_icon_provider, UNDEFINED_QDATE,
         choose_files, error_dialog, choose_images)
-from calibre.utils.date import local_tz, qt_to_dt
+from calibre.utils.date import (local_tz, qt_to_dt, as_local_time,
+        UNDEFINED_DATE)
 from calibre import strftime
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.customize.ui import run_plugins_on_import
@@ -125,6 +126,9 @@ class TitleEdit(EnLineEdit):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.dialog = None
+
 class TitleSortEdit(TitleEdit):
 
     TITLE_ATTR = 'title_sort'
@@ -150,6 +154,7 @@ class TitleSortEdit(TitleEdit):
         self.title_edit.textChanged.connect(self.update_state)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
         autogen_button.clicked.connect(self.auto_generate)
         self.update_state()
 
@@ -168,6 +173,9 @@ class TitleSortEdit(TitleEdit):
 
     def auto_generate(self, *args):
         self.current_val = title_sort(self.title_edit.current_val)
+        self.title_edit.textChanged.disconnect()
+        self.textChanged.disconnect()
+        self.autogen_button.clicked.disconnect()
 
 # }}}
 
@@ -185,6 +193,7 @@ class AuthorsEdit(MultiCompleteComboBox):
         self.setWhatsThis(self.TOOLTIP)
         self.setEditable(True)
         self.setSizeAdjustPolicy(self.AdjustToMinimumContentsLengthWithIcon)
+        self.manage_authors_signal = manage_authors
         manage_authors.triggered.connect(self.manage_authors)
 
     def manage_authors(self):
@@ -269,6 +278,10 @@ class AuthorsEdit(MultiCompleteComboBox):
 
         return property(fget=fget, fset=fset)
 
+    def break_cycles(self):
+        self.db = self.dialog = None
+        self.manage_authors_signal.triggered.disconnect()
+
 class AuthorSortEdit(EnLineEdit):
 
     TOOLTIP = _('Specify how the author(s) of this book should be sorted. '
@@ -297,6 +310,10 @@ class AuthorSortEdit(EnLineEdit):
         self.authors_edit.editTextChanged.connect(self.update_state_and_val)
         self.textChanged.connect(self.update_state)
 
+        self.autogen_button = autogen_button
+        self.copy_a_to_as_action = copy_a_to_as_action
+        self.copy_as_to_a_action = copy_as_to_a_action
+
         autogen_button.clicked.connect(self.auto_generate)
         copy_a_to_as_action.triggered.connect(self.auto_generate)
         copy_as_to_a_action.triggered.connect(self.copy_to_authors)
@@ -368,6 +385,15 @@ class AuthorSortEdit(EnLineEdit):
         db.set_author_sort(id_, aus, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        self.db = None
+        self.authors_edit.editTextChanged.disconnect()
+        self.textChanged.disconnect()
+        self.autogen_button.clicked.disconnect()
+        self.copy_a_to_as_action.triggered.disconnect()
+        self.copy_as_to_a_action.triggered.disconnect()
+        self.authors_edit = None
+
 # }}}
 
 # Series {{{
@@ -427,6 +453,10 @@ class SeriesEdit(MultiCompleteComboBox):
                                             commit=True, allow_case_change=True)
         return True
 
+    def break_cycles(self):
+        self.dialog = None
+
+
 class SeriesIndexEdit(QDoubleSpinBox):
 
     TOOLTIP = ''
@@ -488,6 +518,11 @@ class SeriesIndexEdit(QDoubleSpinBox):
                 import traceback
                 traceback.print_exc()
 
+    def break_cycles(self):
+        self.series_edit.currentIndexChanged.disconnect()
+        self.series_edit.editTextChanged.disconnect()
+        self.series_edit.lineEdit().editingFinished.disconnect()
+        self.db = self.series_edit = self.dialog = None
 
 # }}}
 
@@ -699,6 +734,8 @@ class FormatsManager(QWidget): # {{{
             if old != prefs['read_file_metadata']:
                 prefs['read_file_metadata'] = old
 
+    def break_cycles(self):
+        self.dialog = None
 # }}}
 
 class Cover(ImageView): # {{{
@@ -860,6 +897,10 @@ class Cover(ImageView): # {{{
                 db.remove_cover(id_, notify=False, commit=False)
         return True
 
+    def break_cycles(self):
+        self.cover_changed.disconnect()
+        self.dialog = self._cdata = self.current_val = self.original_val = None
+
 # }}}
 
 class CommentsEdit(Editor): # {{{
@@ -1211,6 +1252,7 @@ class DateEdit(QDateEdit): # {{{
         def fset(self, val):
             if val is None:
                 val = UNDEFINED_DATE
+            val = as_local_time(val)
             self.setDate(QDate(val.year, val.month, val.day))
         return property(fget=fget, fset=fset)
 

src/calibre/gui2/metadata/single.py

@@ -481,6 +481,13 @@ class MetadataSingleDialogBase(ResizableDialog):
             x = getattr(self, b, None)
             if x is not None:
                 disconnect(x.clicked)
+        for widget in self.basic_metadata_widgets:
+            bc = getattr(widget, 'break_cycles', None)
+            if bc is not None and callable(bc):
+                bc()
+        for widget in getattr(self, 'custom_metadata_widgets', []):
+            widget.break_cycles()
+
     # }}}
 
 class Splitter(QSplitter):

src/calibre/gui2/preferences/coloring.py

@@ -72,19 +72,27 @@ class ConditionEditor(QWidget): # {{{
         self.l = l = QGridLayout(self)
         self.setLayout(l)
 
-        self.l1 = l1 = QLabel(_('If the '))
+        texts = _('If the ___ column ___ values')
+        try:
+            one, two, three = texts.split('___')
+        except:
+            one, two, three = 'If the ', ' column ', ' value '
+
+        self.l1 = l1 = QLabel(one)
         l.addWidget(l1, 0, 0)
 
         self.column_box = QComboBox(self)
         l.addWidget(self.column_box, 0, 1)
 
-        self.l2 = l2 = QLabel(_(' column '))
+
+
+        self.l2 = l2 = QLabel(two)
         l.addWidget(l2, 0, 2)
 
         self.action_box = QComboBox(self)
         l.addWidget(self.action_box, 0, 3)
 
-        self.l3 = l3 = QLabel(_(' value '))
+        self.l3 = l3 = QLabel(three)
         l.addWidget(l3, 0, 4)
 
         self.value_box = QLineEdit(self)

src/calibre/gui2/preferences/misc.py

@@ -10,6 +10,7 @@ from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting
 from calibre.gui2.preferences.misc_ui import Ui_Form
 from calibre.gui2 import error_dialog, config, open_local_file, info_dialog
 from calibre.constants import isosx
+from calibre import get_proxies
 
 class WorkersSetting(Setting):
 
@@ -33,6 +34,13 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.user_defined_device_button.clicked.connect(self.user_defined_device)
         self.button_osx_symlinks.clicked.connect(self.create_symlinks)
         self.button_osx_symlinks.setVisible(isosx)
+        proxies = get_proxies(debug=False)
+        txt = _('No proxies used')
+        if proxies:
+            lines = ['<br><code>%s: %s</code>'%(t, p) for t, p in
+                    proxies.iteritems()]
+            txt = _('<b>Using proxies:</b>') + ''.join(lines)
+        self.proxies.setText(txt)
 
     def debug_device_detection(self, *args):
         from calibre.gui2.preferences.device_debug import DebugDevice

src/calibre/gui2/preferences/misc.ui

@@ -118,7 +118,7 @@
      </property>
     </widget>
    </item>
-   <item row="20" column="0">
+   <item row="21" column="0">
     <spacer name="verticalSpacer_9">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -131,6 +131,13 @@
      </property>
     </spacer>
    </item>
+   <item row="10" column="0" colspan="2">
+    <widget class="QLabel" name="proxies">
+     <property name="text">
+      <string/>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources/>

src/calibre/gui2/tag_view.py

@@ -149,7 +149,8 @@ class TagsView(QTreeView): # {{{
                                 hidden_categories=self.hidden_categories,
                                 search_restriction=None,
                                 drag_drop_finished=self.drag_drop_finished,
-                                collapse_model=self.collapse_model)
+                                collapse_model=self.collapse_model,
+                                state_map={})
         self.pane_is_visible = True # because TagsModel.init did a recount
         self.sort_by = sort_by
         self.tag_match = tag_match
@@ -173,6 +174,7 @@ class TagsView(QTreeView): # {{{
             self.made_connections = True
         self.refresh_signal_processed = True
         db.add_listener(self.database_changed)
+        self.expanded.connect(self.item_expanded)
 
     def database_changed(self, event, ids):
         if self.refresh_signal_processed:
@@ -541,6 +543,10 @@ class TagsView(QTreeView): # {{{
         return self.isExpanded(idx)
 
     def recount(self, *args):
+        '''
+        Rebuild the category tree, expand any categories that were expanded,
+        reset the search states, and reselect the current node.
+        '''
         if self.disable_recounting or not self.pane_is_visible:
             return
         self.refresh_signal_processed = True
@@ -548,18 +554,23 @@ class TagsView(QTreeView): # {{{
         if not ci.isValid():
             ci = self.indexAt(QPoint(10, 10))
         path = self.model().path_for_index(ci) if self.is_visible(ci) else None
-        try:
-            if not self.model().refresh(): # categories changed!
-                self.set_new_model()
-                path = None
-        except: #Database connection could be closed if an integrity check is happening
-            pass
+        expanded_categories, state_map = self.model().get_state()
+        self.set_new_model(state_map=state_map)
+        for category in expanded_categories:
+            self.expand(self.model().index_for_category(category))
         self._model.show_item_at_path(path)
 
-    # If the number of user categories changed,  if custom columns have come or
-    # gone, or if columns have been hidden or restored, we must rebuild the
-    # model. Reason: it is much easier than reconstructing the browser tree.
-    def set_new_model(self, filter_categories_by=None):
+    def item_expanded(self, idx):
+        '''
+        Called by the expanded signal
+        '''
+        self.setCurrentIndex(idx)
+
+    def set_new_model(self, filter_categories_by=None, state_map={}):
+        '''
+        There are cases where we need to rebuild the category tree without
+        attempting to reposition the current node.
+        '''
         try:
             old = getattr(self, '_model', None)
             if old is not None:
@@ -569,7 +580,8 @@ class TagsView(QTreeView): # {{{
                                     search_restriction=self.search_restriction,
                                     drag_drop_finished=self.drag_drop_finished,
                                     filter_categories_by=filter_categories_by,
-                                    collapse_model=self.collapse_model)
+                                    collapse_model=self.collapse_model,
+                                    state_map=state_map)
             self.setModel(self._model)
         except:
             # The DB must be gone. Set the model to None and hope that someone
@@ -627,7 +639,8 @@ class TagTreeItem(object): # {{{
             except:
                 pass
         self.parent = self.icon_state_map = self.bold_font = self.tag = \
-                self.icon = self.children = None
+                      self.icon = self.children = self.tooltip = \
+                      self.py_name = self.id_set = self.category_key = None
 
     def __str__(self):
         if self.type == self.ROOT:
@@ -751,7 +764,8 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def __init__(self, db, parent, hidden_categories=None,
             search_restriction=None, drag_drop_finished=None,
-            filter_categories_by=None, collapse_model='disable'):
+            filter_categories_by=None, collapse_model='disable',
+            state_map={}):
         QAbstractItemModel.__init__(self, parent)
 
         # must do this here because 'QPixmap: Must construct a QApplication
@@ -775,10 +789,10 @@ class TagsModel(QAbstractItemModel): # {{{
         self.filter_categories_by = filter_categories_by
         self.collapse_model = collapse_model
 
-        # get_node_tree cannot return None here, because row_map is empty. Note
-        # that get_node_tree can indirectly change the user_categories dict.
+        # Note that _get_category_nodes can indirectly change the
+        # user_categories dict.
 
-        data = self.get_node_tree(config['sort_tags_by'])
+        data = self._get_category_nodes(config['sort_tags_by'])
         gst = db.prefs.get('grouped_search_terms', {})
         self.root_item = TagTreeItem(icon_map=self.icon_state_map)
         self.category_nodes = []
@@ -843,7 +857,7 @@ class TagsModel(QAbstractItemModel): # {{{
                 category_node_map[key] = node
                 last_category_node = node
                 self.category_nodes.append(node)
-        self.refresh(data=data)
+        self._create_node_tree(data, state_map)
 
     def break_cycles(self):
         self.root_item.break_cycles()
@@ -1120,8 +1134,10 @@ class TagsModel(QAbstractItemModel): # {{{
     def set_search_restriction(self, s):
         self.search_restriction = s
 
-    def get_node_tree(self, sort):
-        old_row_map = self.row_map[:]
+    def _get_category_nodes(self, sort):
+        '''
+        Called by __init__. Do not directly call this method.
+        '''
         self.row_map = []
         self.categories = {}
 
@@ -1175,19 +1191,27 @@ class TagsModel(QAbstractItemModel): # {{{
             if category in data: # The search category can come and go
                 self.row_map.append(category)
                 self.categories[category] = tb_categories[category]['name']
-
-        if len(old_row_map) != 0 and len(old_row_map) != len(self.row_map):
-            # A category has been added or removed. We must force a rebuild of
-            # the model
-            return None
         return data
 
     def refresh(self, data=None):
+        '''
+        Here to trap usages of refresh in the old architecture. Can eventually
+        be removed.
+        '''
+        print 'TagsModel: refresh called!'
+        traceback.print_stack()
+        return False
+
+    def _create_node_tree(self, data, state_map):
+        '''
+        Called by __init__. Do not directly call this method.
+        '''
         sort_by = config['sort_tags_by']
+
         if data is None:
-            data = self.get_node_tree(sort_by) # get category data
-        if data is None:
-            return False
+            print '_create_node_tree: no data!'
+            traceback.print_stack()
+            return
 
         collapse = gprefs['tags_browser_collapse_at']
         collapse_model = self.collapse_model
@@ -1353,26 +1377,23 @@ class TagsModel(QAbstractItemModel): # {{{
         # }}}
 
         for category in self.category_nodes:
-            if len(category.children) > 0:
-                child_map = category.children
-                states = [c.tag.state for c in category.child_tags()]
-                names = [(c.tag.name, c.tag.category) for c in category.child_tags()]
-                state_map = dict(izip(names, states))
-                # temporary sub-categories (the partitioning ones) must follow
-                # the permanent sub-categories. This will happen naturally if
-                # the temp ones are added by process_node
-                ctags = [c for c in child_map if
-                         c.type == TagTreeItem.CATEGORY and not c.temporary]
-                start = len(ctags)
-                self.beginRemoveRows(self.createIndex(category.row(), 0, category),
-                                     start, len(child_map)-1)
-                category.children = ctags
-                self.endRemoveRows()
-            else:
-                state_map = {}
+            process_one_node(category, state_map.get(category.py_name, {}))
 
-            process_one_node(category, state_map)
-        return True
+    def get_state(self):
+        state_map = {}
+        expanded_categories = []
+        for row, category in enumerate(self.category_nodes):
+            if self.tags_view.isExpanded(self.index(row, 0, QModelIndex())):
+                expanded_categories.append(category.py_name)
+            states = [c.tag.state for c in category.child_tags()]
+            names = [(c.tag.name, c.tag.category) for c in category.child_tags()]
+            state_map[category.py_name] = dict(izip(names, states))
+        return expanded_categories, state_map
+
+    def index_for_category(self, name):
+        for row, category in enumerate(self.category_nodes):
+            if category.py_name == name:
+                return self.index(row, 0, QModelIndex())
 
     def columnCount(self, parent):
         return 1
@@ -1472,7 +1493,7 @@ class TagsModel(QAbstractItemModel): # {{{
             self.tags_view.tag_item_renamed.emit()
             item.tag.name = val
             self.rename_item_in_all_user_categories(name, key, val)
-            self.refresh() # Should work, because no categories can have disappeared
+            self.refresh_required.emit()
         self.show_item_at_path(path)
         return True
 
@@ -1785,19 +1806,22 @@ class TagsModel(QAbstractItemModel): # {{{
                         return v
         return None
 
-    def show_item_at_path(self, path, box=False):
+    def show_item_at_path(self, path, box=False,
+                          position=QTreeView.PositionAtCenter):
         '''
         Scroll the browser and open categories to show the item referenced by
         path. If possible, the item is placed in the center. If box=True, a
         box is drawn around the item.
         '''
         if path:
-            self.show_item_at_index(self.index_for_path(path), box)
+            self.show_item_at_index(self.index_for_path(path), box=box,
+                                    position=position)
 
-    def show_item_at_index(self, idx, box=False):
+    def show_item_at_index(self, idx, box=False,
+                           position=QTreeView.PositionAtCenter):
         if idx.isValid():
             self.tags_view.setCurrentIndex(idx)
-            self.tags_view.scrollTo(idx, QTreeView.PositionAtCenter)
+            self.tags_view.scrollTo(idx, position)
             if box:
                 tag_item = idx.internalPointer()
                 tag_item.boxed = True

src/calibre/library/database2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid, tempfile
+        json, uuid, tempfile, hashlib
 import threading, random
 from itertools import repeat
 from math import ceil
@@ -1122,9 +1122,45 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return self.format_abspath(index, format, index_is_id) is not None
 
     def format_last_modified(self, id_, fmt):
+        m = self.format_metadata(id_, fmt)
+        if m:
+            return m['mtime']
+
+    def format_metadata(self, id_, fmt):
         path = self.format_abspath(id_, fmt, index_is_id=True)
+        ans = {}
         if path is not None:
-            return utcfromtimestamp(os.stat(path).st_mtime)
+            stat = os.stat(path)
+            ans['size'] = stat.st_size
+            ans['mtime'] = utcfromtimestamp(stat.st_mtime)
+        return ans
+
+    def format_hash(self, id_, fmt):
+        path = self.format_abspath(id_, fmt, index_is_id=True)
+        if path is None:
+            raise NoSuchFormat('Record %d has no fmt: %s'%(id_, fmt))
+        sha = hashlib.sha256()
+        with lopen(path, 'rb') as f:
+            while True:
+                raw = f.read(SPOOL_SIZE)
+                sha.update(raw)
+                if len(raw) < SPOOL_SIZE:
+                    break
+        return sha.hexdigest()
+
+    def format_path(self, index, fmt, index_is_id=False):
+        '''
+        This method is intended to be used only in those rare situations, like
+        Drag'n Drop, when you absolutely need the path to the original file.
+        Otherwise, use format(..., as_path=True).
+
+        Note that a networked backend will always return None.
+        '''
+        path = self.format_abspath(index, fmt, index_is_id=index_is_id)
+        if path is None:
+            id_ = index if index_is_id else self.id(index)
+            raise NoSuchFormat('Record %d has no format: %s'%(id_, fmt))
+        return path
 
     def format_abspath(self, index, format, index_is_id=False):
         '''

src/calibre/manual/conversion.rst

@@ -633,6 +633,7 @@ TXT input supports a number of options to differentiate how paragraphs are detec
     :guilabel:`Formatting Style: None`
         Applies no special formatting to the text, the document is converted to html with no other changes.
 
+.. _pdfconversion:
 
 Convert PDF documents
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/faq.rst

@@ -35,29 +35,11 @@ What are the best source formats to convert?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order of decreasing preference: LIT, MOBI, EPUB, FB2, HTML, PRC, RTF, PDB, TXT, PDF
 
-Why does the PDF conversion lose some images/tables?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The PDF conversion tries to extract the text and images from the PDF file and convert them to and HTML based ebook. Some PDF files have images in a format that cannot be extracted (vector images). All tables
-are also represented as vector diagrams, thus they cannot be extracted.
+I converted a PDF file, but the result has various problems?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-How do I convert a collection of HTML files in a specific order?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
+PDF is a terrible format to convert from. For a list of the various issues you will encounter when converting PDF, see: :ref:`pdfconversion`.
 
-   <html>
-      <body>
-        <h1>Table of Contents</h1>
-        <p style="text-indent:0pt">
-           <a href="file1.html">First File</a><br/>
-           <a href="file2.html">Second File</a><br/>
-           .
-           .
-           .
-        </p>
-      </body>
-   </html>
-
-Then just add this HTML file to the GUI and use the convert button to create your ebook.
 
 .. _char-encoding-faq:
 
@@ -85,6 +67,26 @@ If you have a hand edited TOC in the input document, you can use the TOC detecti
 
 Finally, I encourage you to ditch the content TOC and only have a metadata TOC in your ebooks. Metadata TOCs will give the people reading your ebooks a much superior navigation experience (except on the Kindle, where they are essentially the same as a content TOC).
 
+How do I convert a collection of HTML files in a specific order?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
+
+   <html>
+      <body>
+        <h1>Table of Contents</h1>
+        <p style="text-indent:0pt">
+           <a href="file1.html">First File</a><br/>
+           <a href="file2.html">Second File</a><br/>
+           .
+           .
+           .
+        </p>
+      </body>
+   </html>
+
+Then just add this HTML file to the GUI and use the convert button to create your ebook.
+
+
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features:

src/calibre/utils/date.py

@@ -123,6 +123,14 @@ def isoformat(date_time, assume_utc=False, as_utc=True, sep='T'):
     date_time = date_time.astimezone(_utc_tz if as_utc else _local_tz)
     return unicode(date_time.isoformat(sep))
 
+def as_local_time(date_time, assume_utc=True):
+    if not hasattr(date_time, 'tzinfo'):
+        return date_time
+    if date_time.tzinfo is None:
+        date_time = date_time.replace(tzinfo=_utc_tz if assume_utc else
+                _local_tz)
+    return date_time.astimezone(_local_tz)
+
 def now():
     return datetime.now().replace(tzinfo=_local_tz)
 

src/calibre/utils/formatter_functions.py

@@ -12,7 +12,7 @@ import inspect, re, traceback
 
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp, sort_key
-from calibre.utils.date import parse_date, format_date
+from calibre.utils.date import parse_date, format_date, now, UNDEFINED_DATE
 
 
 class FormatterFunctions(object):
@@ -579,7 +579,7 @@ class BuiltinSubitems(BuiltinFormatterFunction):
 class BuiltinFormatDate(BuiltinFormatterFunction):
     name = 'format_date'
     arg_count = 2
-    category = 'Get values from metadata'
+    category = 'Date functions'
     __doc__ = doc = _('format_date(val, format_string) -- format the value, '
             'which must be a date, using the format_string, returning a string. '
             'The formatting codes are: '
@@ -754,6 +754,39 @@ class BuiltinMergeLists(BuiltinFormatterFunction):
                 res.append(i)
         return ', '.join(sorted(res, key=sort_key))
 
+class BuiltinToday(BuiltinFormatterFunction):
+    name = 'today'
+    arg_count = 0
+    category = 'Date functions'
+    __doc__ = doc = _('today() -- '
+            'return a date string for today. This value is designed for use in '
+            'format_date or days_between, but can be manipulated like any '
+            'other string. The date is in ISO format.')
+    def evaluate(self, formatter, kwargs, mi, locals):
+        return format_date(now(), 'iso')
+
+class BuiltinDaysBetween(BuiltinFormatterFunction):
+    name = 'days_between'
+    arg_count = 2
+    category = 'Date functions'
+    __doc__ = doc = _('days_between(date1, date2) -- '
+            'return the number of days between date1 and date2. The number is '
+            'positive if date1 is greater than date2, otherwise negative. If '
+            'either date1 or date2 are not dates, the function returns the '
+            'empty string.')
+    def evaluate(self, formatter, kwargs, mi, locals, date1, date2):
+        try:
+            d1 = parse_date(date1)
+            if d1 == UNDEFINED_DATE:
+                return ''
+            d2 = parse_date(date2)
+            if d2 == UNDEFINED_DATE:
+                return ''
+        except:
+            return ''
+        i = d1 - d2
+        return str(i.days)
+
 
 builtin_add         = BuiltinAdd()
 builtin_and         = BuiltinAnd()
@@ -763,6 +796,7 @@ builtin_capitalize  = BuiltinCapitalize()
 builtin_cmp         = BuiltinCmp()
 builtin_contains    = BuiltinContains()
 builtin_count       = BuiltinCount()
+builtin_days_between= BuiltinDaysBetween()
 builtin_divide      = BuiltinDivide()
 builtin_eval        = BuiltinEval()
 builtin_first_non_empty = BuiltinFirstNonEmpty()
@@ -795,6 +829,7 @@ builtin_switch      = BuiltinSwitch()
 builtin_template    = BuiltinTemplate()
 builtin_test        = BuiltinTest()
 builtin_titlecase   = BuiltinTitlecase()
+builtin_today       = BuiltinToday()
 builtin_uppercase   = BuiltinUppercase()
 
 class FormatterUserFunction(FormatterFunction):

src/calibre/utils/ipc/proxy.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from threading import Thread
+from multiprocessing.connection import arbitrary_address, Listener
+
+from calibre.constants import iswindows
+
+class Server(Thread):
+
+    def __init__(self):
+        Thread.__init__(self)
+        self.daemon = True
+
+        self.auth_key = os.urandom(32)
+        self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        if iswindows and self.address[1] == ':':
+            self.address = self.address[2:]
+        self.listener = Listener(address=self.address,
+                authkey=self.auth_key, backlog=4)
+
+        self.keep_going = True
+
+    def stop(self):
+        self.keep_going = False
+        try:
+            self.listener.close()
+        except:
+            pass
+
+    def run(self):
+        while self.keep_going:
+            try:
+                conn = self.listener.accept()
+                self.handle_client(conn)
+            except:
+                pass
+
+

src/calibre/utils/mem.py

@@ -8,61 +8,157 @@ __docformat__ = 'restructuredtext en'
 '''
 Measure memory usage of the current process.
 
-The key function is memory() which returns the current memory usage in bytes.
+The key function is memory() which returns the current memory usage in MB.
 You can pass a number to memory and it will be subtracted from the returned
 value.
 '''
 
-import gc, os
+import gc, os, re
 
 from calibre.constants import iswindows, islinux
 
 if islinux:
-    ## {{{ http://code.activestate.com/recipes/286222/ (r1)
+    # Taken, with thanks, from:
+    # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
 
-    _proc_status = '/proc/%d/status' % os.getpid()
+    def permute(args):
+        ret = []
+        if args:
+            first = args.pop(0)
+            for y in permute(args):
+                for x in first:
+                    ret.append(x + y)
+        else:
+            ret.append('')
+        return ret
 
-    _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
-            'KB': 1024.0, 'MB': 1024.0*1024.0}
+    def parsed_groups(match, *types):
+        groups = match.groups()
+        assert len(groups) == len(types)
+        return tuple([type(group) for group, type in zip(groups, types)])
 
-    def _VmB(VmKey):
-        '''Private.
-        '''
-        global _proc_status, _scale
-        # get pseudo file  /proc/<pid>/status
-        try:
-            t = open(_proc_status)
-            v = t.read()
-            t.close()
-        except:
-            return 0.0  # non-Linux?
-        # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
-        i = v.index(VmKey)
-        v = v[i:].split(None, 3)  # whitespace
-        if len(v) < 3:
-            return 0.0  # invalid format?
-        # convert Vm value to bytes
-        return float(v[1]) * _scale[v[2]]
+    class VMA(dict):
+        def __init__(self, *args):
+            (self.start, self.end, self.perms, self.offset,
+            self.major, self.minor, self.inode, self.filename) = args
 
+    def parse_smaps(pid):
+        with open('/proc/%s/smaps'%pid, 'r') as maps:
+            hex = lambda s: int(s, 16)
+
+            ret = []
+            header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
+                                r'(..):(..) (\d+) *(.*)$')
+            detail = re.compile(r'^(.*): +(\d+) kB')
+            for line in maps:
+                m = header.match(line)
+                if m:
+                    vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
+                    ret.append(vma)
+                else:
+                    m = detail.match(line)
+                    if m:
+                        k, v = parsed_groups(m, str, int)
+                        assert k not in vma
+                        vma[k] = v
+                    else:
+                        print 'unparseable line:', line
+            return ret
+
+    perms = permute(['r-', 'w-', 'x-', 'ps'])
+
+    def make_summary_dicts(vmas):
+        mapped = {}
+        anon = {}
+        for d in mapped, anon:
+            # per-perm
+            for k in perms:
+                d[k] = {}
+                d[k]['Size'] = 0
+                for y in 'Shared', 'Private':
+                    d[k][y] = {}
+                    for z in 'Clean', 'Dirty':
+                        d[k][y][z] = 0
+            # totals
+            for y in 'Shared', 'Private':
+                d[y] = {}
+                for z in 'Clean', 'Dirty':
+                    d[y][z] = 0
+
+        for vma in vmas:
+            if vma.major == '00' and vma.minor == '00':
+                d = anon
+            else:
+                d = mapped
+            for y in 'Shared', 'Private':
+                for z in 'Clean', 'Dirty':
+                    d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
+                    d[y][z] += vma.get(y + '_' + z, 0)
+            d[vma.perms]['Size'] += vma.get('Size', 0)
+        return mapped, anon
+
+    def values(d, args):
+        if args:
+            ret = ()
+            first = args[0]
+            for k in first:
+                ret += values(d[k], args[1:])
+            return ret
+        else:
+            return (d,)
+
+    def print_summary(dicts_and_titles):
+        def desc(title, perms):
+            ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
+                ('Anonymous', 'rwxp'): 'Writable code (stack)',
+                ('Mapped', 'r-xp'): 'Code',
+                ('Mapped', 'rwxp'): 'Writable code (jump tables)',
+                ('Mapped', 'r--p'): 'Read-only data',
+                ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
+            if ret:
+                return '  -- ' + ret
+            else:
+                return ''
+
+        for d, title in dicts_and_titles:
+            print title, 'memory:'
+            print '               Shared            Private'
+            print '           Clean    Dirty    Clean    Dirty'
+            for k in perms:
+                if d[k]['Size']:
+                    print ('    %s %7d  %7d  %7d  %7d%s'
+                        % ((k,)
+                            + values(d[k], (('Shared', 'Private'),
+                                            ('Clean', 'Dirty')))
+                            + (desc(title, k),)))
+            print ('   total %7d  %7d  %7d  %7d'
+                % values(d, (('Shared', 'Private'),
+                                ('Clean', 'Dirty'))))
+
+        print '   ' + '-' * 40
+        print ('   total %7d  %7d  %7d  %7d'
+            % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles]))))
+
+    def print_stats(pid=None):
+        if pid is None:
+            pid = os.getpid()
+        vmas = parse_smaps(pid)
+        mapped, anon = make_summary_dicts(vmas)
+        print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
 
     def linux_memory(since=0.0):
-        '''Return memory usage in bytes.
-        '''
-        return _VmB('VmSize:') - since
+        vmas = parse_smaps(os.getpid())
+        mapped, anon = make_summary_dicts(vmas)
+        dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
+        totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles])))
+        return (totals[-1]/1024.) - since
 
-
-    def resident(since=0.0):
-        '''Return resident memory usage in bytes.
-        '''
-        return _VmB('VmRSS:') - since
-
-
-    def stacksize(since=0.0):
-        '''Return stack size in bytes.
-        '''
-        return _VmB('VmStk:') - since
-    ## end of http://code.activestate.com/recipes/286222/ }}}
     memory = linux_memory
+
 elif iswindows:
     import win32process
     import win32con
@@ -95,7 +191,7 @@ elif iswindows:
 
     def win_memory(since=0.0):
         info = meminfo(get_handle(os.getpid()))
-        return info['WorkingSetSize'] - since
+        return (info['WorkingSetSize']/1024.**2) - since
 
     memory = win_memory
 
@@ -112,6 +208,8 @@ def gc_histogram():
 def diff_hists(h1, h2):
     """Prints differences between two results of gc_histogram()."""
     for k in h1:
+        if k not in h2:
+            h2[k] = 0
         if h1[k] != h2[k]:
             print "%s: %d -> %d (%s%d)" % (
                 k, h1[k], h2[k], h2[k] > h1[k] and "+" or "", h2[k] - h1[k])