Merge from trunk

src/calibre/devices/apple/driver.py

@@ -5,7 +5,7 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, ctypes, datetime, os, re, shutil, subprocess, sys, tempfile, time
+import cStringIO, ctypes, datetime, os, re, sys, tempfile, time
 from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image, confirm_config_name
 from calibre.constants import isosx, iswindows
@@ -13,8 +13,7 @@ from calibre.devices.errors import OpenFeedback, UserFeedback
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-from calibre.ebooks.metadata import authors_to_string, MetaInformation, \
+from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort
-    title_sort
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
@@ -165,8 +164,12 @@ class ITUNES(DriverBase):
         settings()
         set_progress_reporter()
         upload_books()
-         _get_fpath()
+         _remove_existing_copy()
-          _update_epub_metadata()
+          _remove_from_device()
+          _remove_from_iTunes()
+         _add_new_copy()
+          _add_library_book()
+         _update_iTunes_metadata()
         add_books_to_metadata()
         use_plugboard_ext()
         set_plugboard()
@@ -183,7 +186,7 @@ class ITUNES(DriverBase):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (1,0,0)
+    version        = (1,1,0)
 
     DISPLAY_DISABLE_DIALOG = "display_disable_apple_driver_dialog"
 
@@ -278,7 +281,6 @@ class ITUNES(DriverBase):
     description_prefix = "added by calibre"
     ejected = False
     iTunes= None
-    iTunes_media = None
     library_orphans = None
     log = Log()
     manual_sync_mode = False
@@ -414,11 +416,11 @@ class ITUNES(DriverBase):
                             this_book.datetime = parse_date(str(book.date_added())).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                         this_book.size = book.size()
                         this_book.uuid = book.composer()
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -453,10 +455,10 @@ class ITUNES(DriverBase):
                                 this_book.datetime = parse_date(str(book.DateAdded)).timetuple()
                             except:
                                 this_book.datetime = time.gmtime()
-                            this_book.db_id = None
                             this_book.device_collections = []
                             this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                             this_book.size = book.Size
+                            this_book.cid = None
                             # Hack to discover if we're running in GUI environment
                             if self.report_progress is not None:
                                 this_book.thumbnail = self._generate_thumbnail(this_book.path, book)
@@ -492,7 +494,7 @@ class ITUNES(DriverBase):
 
     def can_handle(self, device_info, debug=False):
         '''
-        Unix version of :method:`can_handle_windows`
+        OSX version of :method:`can_handle_windows`
 
         :param device_info: Is a tupe of (vid, pid, bcd, manufacturer, product,
         serial number)
@@ -1022,17 +1024,14 @@ class ITUNES(DriverBase):
 
         if DEBUG:
             self.log.info("ITUNES.upload_books()")
-            self._dump_files(files, header='upload_books()',indent=2)
-            self._dump_update_list(header='upload_books()',indent=2)
 
         if isosx:
-            for (i,file) in enumerate(files):
+            for (i,fpath) in enumerate(files):
-                format = file.rpartition('.')[2].lower()
+                format = fpath.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title,
                                              authors_to_string(metadata[i].authors),
                                              format)
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
                 thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
                 this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
@@ -1063,13 +1062,12 @@ class ITUNES(DriverBase):
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
-                for (i,file) in enumerate(files):
+                for (i,fpath) in enumerate(files):
-                    format = file.rpartition('.')[2].lower()
+                    format = fpath.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title,
                                                  authors_to_string(metadata[i].authors),
                                                  format)
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -1276,24 +1274,59 @@ class ITUNES(DriverBase):
 
     def _add_new_copy(self, fpath, metadata):
         '''
+        fp = cached_book['lib_book'].location().path
+        fp = cached_book['lib_book'].Location
         '''
         if DEBUG:
             self.log.info(" ITUNES._add_new_copy()")
 
+        def _save_last_known_iTunes_storage(lb_added):
+            if isosx:
+                fp = lb_added.location().path
+                index = fp.rfind('/Books') + len('/Books')
+                last_known_iTunes_storage = fp[:index]
+            elif iswindows:
+                fp = lb_added.Location
+                index = fp.rfind('\Books') + len('\Books')
+                last_known_iTunes_storage = fp[:index]
+            dynamic['last_known_iTunes_storage'] = last_known_iTunes_storage
+            self.log.warning("  last_known_iTunes_storage: %s" % last_known_iTunes_storage)
+
         db_added = None
         lb_added = None
 
         if self.manual_sync_mode:
+            '''
+            This is the unsupported direct-connect mode.
+            In an attempt to avoid resetting the iTunes library Media folder, don't try to
+            add the book to iTunes if the last_known_iTunes_storage path is inaccessible.
+            This means that the path has to be set at least once, probably by using
+            'Connect to iTunes' and doing a transfer.
+            '''
+            self.log.warning("  unsupported direct connect mode")
             db_added = self._add_device_book(fpath, metadata)
-            if not getattr(fpath, 'deleted_after_upload', False):
+            last_known_iTunes_storage = dynamic.get('last_known_iTunes_storage', None)
-                lb_added = self._add_library_book(fpath, metadata)
+            if last_known_iTunes_storage is not None:
-                if lb_added:
+                if os.path.exists(last_known_iTunes_storage):
                     if DEBUG:
-                        self.log.info("  file added to Library|Books for iTunes<->iBooks tracking")
+                        self.log.warning("  iTunes storage online, adding to library")
+                    lb_added = self._add_library_book(fpath, metadata)
+                else:
+                    if DEBUG:
+                        self.log.warning("  iTunes storage not online, can't add to library")
+
+            if lb_added:
+                _save_last_known_iTunes_storage(lb_added)
+            if not lb_added and DEBUG:
+                self.log.warn("  failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title)
         else:
             lb_added = self._add_library_book(fpath, metadata)
-            if DEBUG:
+            if lb_added:
-                self.log.info("  file added to Library|Books for pending sync")
+                _save_last_known_iTunes_storage(lb_added)
+            else:
+                raise UserFeedback("iTunes Media folder inaccessible",
+                                   details="Failed to add '%s' to iTunes" % metadata.title,
+                                   level=UserFeedback.WARN)
 
         return db_added, lb_added
 
@@ -1308,8 +1341,10 @@ class ITUNES(DriverBase):
         if metadata.cover:
 
             if format == 'epub':
-                # Pre-shrink cover
+                '''
-                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                Pre-shrink cover
+                self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                '''
                 try:
                     img = PILImage.open(metadata.cover)
                     width = img.size[0]
@@ -1317,8 +1352,8 @@ class ITUNES(DriverBase):
                     scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
                     if scaled:
                         if DEBUG:
-                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
+                            self.log.info("   cover scaled from %sx%s to %sx%s" %
-                                          (metadata.cover,width,height,nwidth,nheight))
+                                          (width,height,nwidth,nheight))
                         img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
                         cd = cStringIO.StringIO()
                         img.convert('RGB').save(cd, 'JPEG')
@@ -1337,9 +1372,11 @@ class ITUNES(DriverBase):
                     return thumb
 
                 if isosx:
-                    # The following commands generate an error, but the artwork does in fact
+                    '''
-                    # get sent to the device.  Seems like a bug in Apple's automation interface?
+                    The following commands generate an error, but the artwork does in fact
-                    # Could also be a problem with the integrity of the cover data?
+                    get sent to the device.  Seems like a bug in Apple's automation interface?
+                    Could also be a problem with the integrity of the cover data?
+                    '''
                     if lb_added:
                         try:
                             lb_added.artworks[1].data_.set(cover_data)
@@ -1362,9 +1399,8 @@ class ITUNES(DriverBase):
                             #ipython(user_ns=locals())
                             pass
 
-
                 elif iswindows:
-                    # Write the data to a real file for Windows iTunes
+                    ''' Write the data to a real file for Windows iTunes '''
                     tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
                     with open(tc,'wb') as tmp_cover:
                         tmp_cover.write(cover_data)
@@ -1423,7 +1459,8 @@ class ITUNES(DriverBase):
 
         this_book = Book(metadata.title, authors_to_string(metadata.authors))
         this_book.datetime = time.gmtime()
-        this_book.db_id = None
+        #this_book.cid = metadata.id
+        this_book.cid = None
         this_book.device_collections = []
         this_book.format = format
         this_book.library_id = lb_added     # ??? GR
@@ -1431,7 +1468,6 @@ class ITUNES(DriverBase):
         this_book.thumbnail = thumb
         this_book.iTunes_id = lb_added      # ??? GR
         this_book.uuid = metadata.uuid
-
         if isosx:
             if lb_added:
                 this_book.size = self._get_device_book_size(fpath, lb_added.size())
@@ -1462,24 +1498,6 @@ class ITUNES(DriverBase):
 
         return this_book
 
-    def _delete_iTunesMetadata_plist(self,fpath):
-        '''
-        Delete the plist file from the file to force recache
-        '''
-        zf = ZipFile(fpath,'a')
-        fnames = zf.namelist()
-        pl_name = 'iTunesMetadata.plist'
-        try:
-            plist = [x for x in fnames if pl_name in x][0]
-        except:
-            plist = None
-        if plist:
-            if DEBUG:
-                self.log.info("  _delete_iTunesMetadata_plist():")
-                self.log.info("    deleting '%s'\n   from '%s'" % (pl_name,fpath))
-                zf.delete(pl_name)
-        zf.close()
-
     def _discover_manual_sync_mode(self, wait=0):
         '''
         Assumes pythoncom for windows
@@ -1664,18 +1682,6 @@ class ITUNES(DriverBase):
         zf.close()
         return (title, author, timestamp)
 
-    def _dump_files(self, files, header=None,indent=0):
-        if header:
-            msg = '\n%sfiles passed to %s:' % (' '*indent,header)
-            self.log.info(msg)
-            self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
-        for file in files:
-            if getattr(file, 'orig_file_path', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.orig_file_path))
-            elif getattr(file, 'name', None) is not None:
-                self.log.info(" %s%s" % (' '*indent,file.name))
-        self.log.info()
-
     def _dump_hex(self, src, length=16):
         '''
         '''
@@ -1699,7 +1705,7 @@ class ITUNES(DriverBase):
         self.log.info()
 
     def _dump_update_list(self,header=None,indent=0):
-        if header:
+        if header and self.update_list:
             msg = '\n%sself.update_list %s' % (' '*indent,header)
             self.log.info(msg)
             self.log.info( "%s%s" % (' '*indent,'-' * len(msg)))
@@ -1718,7 +1724,6 @@ class ITUNES(DriverBase):
                  (' '*indent,
                   ub['title'],
                   ub['author']))
-        self.log.info()
 
     def _find_device_book(self, search):
         '''
@@ -2117,35 +2122,6 @@ class ITUNES(DriverBase):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, format, update_md=False):
-        '''
-        If the database copy will be deleted after upload, we have to
-        use file (the PersistentTemporaryFile), which will be around until
-        calibre exits.
-        If we're using the database copy, delete the plist
-        '''
-        if DEBUG:
-            self.log.info(" ITUNES._get_fpath()")
-
-        fpath = file
-        if not getattr(fpath, 'deleted_after_upload', False):
-            if getattr(file, 'orig_file_path', None) is not None:
-                # Database copy
-                fpath = file.orig_file_path
-                self._delete_iTunesMetadata_plist(fpath)
-            elif getattr(file, 'name', None) is not None:
-                # PTF
-                fpath = file.name
-        else:
-            # Recipe - PTF
-            if DEBUG:
-                self.log.info("   file will be deleted after upload")
-
-        if format == 'epub' and update_md:
-            self._update_epub_metadata(fpath, metadata)
-
-        return fpath
-
     def _get_library_books(self):
         '''
         Populate a dict of paths from iTunes Library|Books
@@ -2349,6 +2325,7 @@ class ITUNES(DriverBase):
                 self.iTunes = appscript.app('iTunes')
                 self.initial_status = 'already running'
 
+            '''
             # Read the current storage path for iTunes media
             cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
             proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
@@ -2359,12 +2336,13 @@ class ITUNES(DriverBase):
             else:
                 self.log.error("  could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
                 self.log.error("  media_dir: %s" % media_dir)
+            '''
+
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [OSX %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
         if iswindows:
@@ -2404,6 +2382,7 @@ class ITUNES(DriverBase):
                              ' iTunes automation interface non-responsive, ' +
                              'recommend reinstalling iTunes')
 
+            '''
             # Read the current storage path for iTunes media from the XML file
             media_dir = ''
             string = None
@@ -2422,13 +2401,13 @@ class ITUNES(DriverBase):
                     self.log.error("  '%s' not found" % media_dir)
                 else:
                     self.log.error("  no media dir found: string: %s" % string)
+            '''
 
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
                 self.log.info("  [Windows %s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
-                self.log.info("  iTunes_media: %s" % self.iTunes_media)
                 self.log.info("  calibre_library_path: %s" % self.calibre_library_path)
 
     def _purge_orphans(self,library_books, cached_books):
@@ -2478,13 +2457,14 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                    self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_device(self.cached_books[book])
+
                     if DEBUG:
                         self.log.info( "  deleting device book '%s'" % (metadata.title))
-                    if not getattr(file, 'deleted_after_upload', False):
+                    self._remove_from_device(self.cached_books[book])
-                        self._remove_from_iTunes(self.cached_books[book])
+
                     if DEBUG:
                         self.log.info("  deleting library book '%s'" % metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2497,9 +2477,9 @@ class ITUNES(DriverBase):
                    (self.cached_books[book]['title'] == metadata.title and \
                     self.cached_books[book]['author'] == authors_to_string(metadata.authors)):
                     self.update_list.append(self.cached_books[book])
-                    self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
                         self.log.info( "  deleting library book '%s'" %  metadata.title)
+                    self._remove_from_iTunes(self.cached_books[book])
                     break
             else:
                 if DEBUG:
@@ -2530,96 +2510,105 @@ class ITUNES(DriverBase):
 
     def _remove_from_iTunes(self, cached_book):
         '''
-        iTunes does not delete books from storage when removing from database
+        iTunes does not delete books from storage when removing from database via automation
-        We only want to delete stored copies if the file is stored in iTunes
-        We don't want to delete files stored outside of iTunes.
-        Also confirm that storage_path does not point into calibre's storage.
         '''
         if DEBUG:
             self.log.info(" ITUNES._remove_from_iTunes():")
 
         if isosx:
+            ''' Manually remove the book from iTunes storage '''
             try:
-                storage_path = os.path.split(cached_book['lib_book'].location().path)
+                fp = cached_book['lib_book'].location().path
-                if cached_book['lib_book'].location().path.startswith(self.iTunes_media) and \
-                   not storage_path[0].startswith(prefs['library_path']):
-                    title_storage_path = storage_path[0]
                 if DEBUG:
-                        self.log.info("  removing title_storage_path: %s" % title_storage_path)
+                    self.log.info("  processing %s" % fp)
+                if fp.startswith(prefs['library_path']):
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
+                else:
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
                         try:
-                        shutil.rmtree(title_storage_path)
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
                         except:
-                        self.log.info("  '%s' not empty" % title_storage_path)
-
-                    # Clean up title/author directories
-                    author_storage_path = os.path.split(title_storage_path)[0]
-                    self.log.info("  author_storage_path: %s" % author_storage_path)
                             author_files = os.listdir(author_storage_path)
                             if '.DS_Store' in author_files:
                                 author_files.pop(author_files.index('.DS_Store'))
                             if not author_files:
-                        shutil.rmtree(author_storage_path)
+                                os.rmdir(author_storage_path)
                                 if DEBUG:
-                            self.log.info("  removing empty author_storage_path")
+                                    self.log.info("   removing empty author directory")
+                            '''
                             else:
                                 if DEBUG:
-                            self.log.info("  author_storage_path not empty (%d objects):" % len(author_files))
+                                    self.log.info("   author_storage_path not empty:")
                                     self.log.info("   %s" % '\n'.join(author_files))
+                            '''
                     else:
-                    self.log.info("  '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
 
             except:
                 # We get here if there was an error with .location().path
                 if DEBUG:
-                    self.log.info("   '%s' not in iTunes storage" % cached_book['title'])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
 
+            # Delete the book from the iTunes database
             try:
                 self.iTunes.delete(cached_book['lib_book'])
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
         elif iswindows:
             '''
             Assume we're wrapped in a pythoncom
             Windows stores the book under a common author directory, so we just delete the .epub
             '''
+            fp = None
             try:
                 book = cached_book['lib_book']
-                path = book.Location
+                fp = book.Location
             except:
                 book = self._find_library_book(cached_book)
                 if book:
-                    path = book.Location
+                    fp = book.Location
 
             if book:
-                if self.iTunes_media and path.startswith(self.iTunes_media) and \
-                   not path.startswith(prefs['library_path']):
-                    storage_path = os.path.split(path)
                 if DEBUG:
-                        self.log.info("   removing '%s' at %s" %
+                    self.log.info("  processing %s" % fp)
-                            (cached_book['title'], path))
+                if fp.startswith(prefs['library_path']):
+                    self.log.info("  '%s' stored in calibre database, not removed" % cached_book['title'])
+                else:
+                    if os.path.exists(fp):
+                        os.remove(fp)
+                        if DEBUG:
+                            self.log.info("   deleting from iTunes storage")
+                        author_storage_path = os.path.split(fp)[0]
                         try:
-                        os.remove(path)
+                            os.rmdir(author_storage_path)
+                            if DEBUG:
+                                self.log.info("   removing empty author directory")
                         except:
-                        self.log.warning("   '%s' not in iTunes storage" % path)
+                            pass
-                    try:
+                    else:
-                        os.rmdir(storage_path[0])
+                        self.log.info("   '%s' does not exist at storage location" % cached_book['title'])
-                        self.log.info("   removed folder '%s'" % storage_path[0])
+            else:
-                    except:
+                if DEBUG:
-                        self.log.info("   folder '%s' not found or not empty" % storage_path[0])
+                    self.log.info("   '%s' not found in iTunes storage" % cached_book['title'])
 
-                    # Delete from iTunes database
+            # Delete the book from the iTunes database
-                else:
-                    self.log.info("   '%s' (stored external to iTunes, no files deleted)" % cached_book['title'])
-            else:
-                if DEBUG:
-                    self.log.info("   '%s' not found in iTunes" % cached_book['title'])
             try:
                 book.Delete()
+                if DEBUG:
+                    self.log.info("   removing from iTunes database")
             except:
                 if DEBUG:
-                    self.log.info("   unable to remove '%s' from iTunes" % cached_book['title'])
+                    self.log.info("   unable to remove from iTunes database")
 
     def title_sorter(self, title):
         return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip()
@@ -2798,7 +2787,7 @@ class ITUNES(DriverBase):
             if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]:
                 if DEBUG:
                     self.log.info(" ITUNES._update_iTunes_metadata()")
-                    self.log.info("   using Series name as Genre")
+                    self.log.info("   using Series name '%s' as Genre" % metadata_x.series)
 
                 # Format the index as a sort key
                 index = metadata_x.series_index
@@ -2978,8 +2967,8 @@ class ITUNES(DriverBase):
             newmi = book.deepcopy_metadata()
             newmi.template_to_attribute(book, pb)
             if pb is not None and DEBUG:
-                self.log.info(" transforming %s using %s:" % (format, pb))
+                #self.log.info(" transforming %s using %s:" % (format, pb))
-                self.log.info("       title: %s %s" % (book.title, ">>> %s" %
+                self.log.info("       title: '%s' %s" % (book.title, ">>> '%s'" %
                                            newmi.title if book.title != newmi.title else ''))
                 self.log.info("  title_sort: %s %s" % (book.title_sort, ">>> %s" %
                                            newmi.title_sort if book.title_sort != newmi.title_sort else ''))
@@ -3083,12 +3072,12 @@ class ITUNES_ASYNC(ITUNES):
                         this_book.datetime = parse_date(str(library_books[book].date_added())).timetuple()
                     except:
                         this_book.datetime = time.gmtime()
-                    this_book.db_id = None
                     this_book.device_collections = []
                     #this_book.library_id = library_books[this_book.path] if this_book.path in library_books else None
                     this_book.library_id = library_books[book]
                     this_book.size = library_books[book].size()
                     this_book.uuid = library_books[book].composer()
+                    this_book.cid = None
                     # Hack to discover if we're running in GUI environment
                     if self.report_progress is not None:
                         this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])
@@ -3124,11 +3113,11 @@ class ITUNES_ASYNC(ITUNES):
                             this_book.datetime = parse_date(str(library_books[book].DateAdded)).timetuple()
                         except:
                             this_book.datetime = time.gmtime()
-                        this_book.db_id = None
                         this_book.device_collections = []
                         this_book.library_id = library_books[book]
                         this_book.size = library_books[book].Size
                         this_book.uuid = library_books[book].Composer
+                        this_book.cid = None
                         # Hack to discover if we're running in GUI environment
                         if self.report_progress is not None:
                             this_book.thumbnail = self._generate_thumbnail(this_book.path, library_books[book])

src/calibre/gui2/actions/edit_metadata.py

@@ -170,10 +170,6 @@ class EditMetadataAction(InterfaceAction):
                 list(range(self.gui.library_view.model().rowCount(QModelIndex())))
             current_row = row_list.index(cr)
 
-        from calibre.utils.mem import memory
-
-        import gc
-        print 'start of edit metadata:', memory()/1024**2
         changed, rows_to_refresh = self.do_edit_metadata(row_list, current_row)
 
         m = self.gui.library_view.model()
@@ -188,9 +184,6 @@ class EditMetadataAction(InterfaceAction):
                 self.gui.cover_flow.dataChanged()
             m.current_changed(current, previous)
             self.gui.tags_view.recount()
-        for i in range(5): gc.collect();
-        print 'end of edit metadata:', memory()/1024**2
-        print
 
     def do_edit_metadata(self, row_list, current_row):
         from calibre.gui2.metadata.single import edit_metadata

src/calibre/gui2/tag_view.py

@@ -1184,8 +1184,6 @@ class TagsModel(QAbstractItemModel): # {{{
         return data
 
     def refresh(self, data=None):
-        from calibre.utils.mem import memory
-        print 'start of refresh:', memory()/1024**2
         sort_by = config['sort_tags_by']
         if data is None:
             data = self.get_node_tree(sort_by) # get category data
@@ -1378,7 +1376,6 @@ class TagsModel(QAbstractItemModel): # {{{
                 state_map = {}
 
             process_one_node(category, state_map)
-        print 'end of refresh:', memory()/1024**2
         return True
 
     def columnCount(self, parent):

src/calibre/utils/mem.py

@@ -8,61 +8,157 @@ __docformat__ = 'restructuredtext en'
 '''
 Measure memory usage of the current process.
 
-The key function is memory() which returns the current memory usage in bytes.
+The key function is memory() which returns the current memory usage in MB.
 You can pass a number to memory and it will be subtracted from the returned
 value.
 '''
 
-import gc, os
+import gc, os, re
 
 from calibre.constants import iswindows, islinux
 
 if islinux:
-    ## {{{ http://code.activestate.com/recipes/286222/ (r1)
+    # Taken, with thanks, from:
+    # http://wingolog.org/archives/2007/11/27/reducing-the-footprint-of-python-applications
 
-    _proc_status = '/proc/%d/status' % os.getpid()
+    def permute(args):
+        ret = []
+        if args:
+            first = args.pop(0)
+            for y in permute(args):
+                for x in first:
+                    ret.append(x + y)
+        else:
+            ret.append('')
+        return ret
 
-    _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
+    def parsed_groups(match, *types):
-            'KB': 1024.0, 'MB': 1024.0*1024.0}
+        groups = match.groups()
+        assert len(groups) == len(types)
+        return tuple([type(group) for group, type in zip(groups, types)])
 
-    def _VmB(VmKey):
+    class VMA(dict):
-        '''Private.
+        def __init__(self, *args):
-        '''
+            (self.start, self.end, self.perms, self.offset,
-        global _proc_status, _scale
+            self.major, self.minor, self.inode, self.filename) = args
-        # get pseudo file  /proc/<pid>/status
-        try:
-            t = open(_proc_status)
-            v = t.read()
-            t.close()
-        except:
-            return 0.0  # non-Linux?
-        # get VmKey line e.g. 'VmRSS:  9999  kB\n ...'
-        i = v.index(VmKey)
-        v = v[i:].split(None, 3)  # whitespace
-        if len(v) < 3:
-            return 0.0  # invalid format?
-        # convert Vm value to bytes
-        return float(v[1]) * _scale[v[2]]
 
+    def parse_smaps(pid):
+        with open('/proc/%s/smaps'%pid, 'r') as maps:
+            hex = lambda s: int(s, 16)
+
+            ret = []
+            header = re.compile(r'^([0-9a-f]+)-([0-9a-f]+) (....) ([0-9a-f]+) '
+                                r'(..):(..) (\d+) *(.*)$')
+            detail = re.compile(r'^(.*): +(\d+) kB')
+            for line in maps:
+                m = header.match(line)
+                if m:
+                    vma = VMA(*parsed_groups(m, hex, hex, str, hex, str, str, int, str))
+                    ret.append(vma)
+                else:
+                    m = detail.match(line)
+                    if m:
+                        k, v = parsed_groups(m, str, int)
+                        assert k not in vma
+                        vma[k] = v
+                    else:
+                        print 'unparseable line:', line
+            return ret
+
+    perms = permute(['r-', 'w-', 'x-', 'ps'])
+
+    def make_summary_dicts(vmas):
+        mapped = {}
+        anon = {}
+        for d in mapped, anon:
+            # per-perm
+            for k in perms:
+                d[k] = {}
+                d[k]['Size'] = 0
+                for y in 'Shared', 'Private':
+                    d[k][y] = {}
+                    for z in 'Clean', 'Dirty':
+                        d[k][y][z] = 0
+            # totals
+            for y in 'Shared', 'Private':
+                d[y] = {}
+                for z in 'Clean', 'Dirty':
+                    d[y][z] = 0
+
+        for vma in vmas:
+            if vma.major == '00' and vma.minor == '00':
+                d = anon
+            else:
+                d = mapped
+            for y in 'Shared', 'Private':
+                for z in 'Clean', 'Dirty':
+                    d[vma.perms][y][z] += vma.get(y + '_' + z, 0)
+                    d[y][z] += vma.get(y + '_' + z, 0)
+            d[vma.perms]['Size'] += vma.get('Size', 0)
+        return mapped, anon
+
+    def values(d, args):
+        if args:
+            ret = ()
+            first = args[0]
+            for k in first:
+                ret += values(d[k], args[1:])
+            return ret
+        else:
+            return (d,)
+
+    def print_summary(dicts_and_titles):
+        def desc(title, perms):
+            ret = {('Anonymous', 'rw-p'): 'Data (malloc, mmap)',
+                ('Anonymous', 'rwxp'): 'Writable code (stack)',
+                ('Mapped', 'r-xp'): 'Code',
+                ('Mapped', 'rwxp'): 'Writable code (jump tables)',
+                ('Mapped', 'r--p'): 'Read-only data',
+                ('Mapped', 'rw-p'): 'Data'}.get((title, perms), None)
+            if ret:
+                return '  -- ' + ret
+            else:
+                return ''
+
+        for d, title in dicts_and_titles:
+            print title, 'memory:'
+            print '               Shared            Private'
+            print '           Clean    Dirty    Clean    Dirty'
+            for k in perms:
+                if d[k]['Size']:
+                    print ('    %s %7d  %7d  %7d  %7d%s'
+                        % ((k,)
+                            + values(d[k], (('Shared', 'Private'),
+                                            ('Clean', 'Dirty')))
+                            + (desc(title, k),)))
+            print ('   total %7d  %7d  %7d  %7d'
+                % values(d, (('Shared', 'Private'),
+                                ('Clean', 'Dirty'))))
+
+        print '   ' + '-' * 40
+        print ('   total %7d  %7d  %7d  %7d'
+            % tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles]))))
+
+    def print_stats(pid=None):
+        if pid is None:
+            pid = os.getpid()
+        vmas = parse_smaps(pid)
+        mapped, anon = make_summary_dicts(vmas)
+        print_summary(((mapped, "Mapped"), (anon, "Anonymous")))
 
     def linux_memory(since=0.0):
-        '''Return memory usage in bytes.
+        vmas = parse_smaps(os.getpid())
-        '''
+        mapped, anon = make_summary_dicts(vmas)
-        return _VmB('VmSize:') - since
+        dicts_and_titles = ((mapped, "Mapped"), (anon, "Anonymous"))
+        totals = tuple(map(sum, zip(*[values(d, (('Shared', 'Private'),
+                                                ('Clean', 'Dirty')))
+                                    for d, title in dicts_and_titles])))
+        return (totals[-1]/1024.) - since
 
-
-    def resident(since=0.0):
-        '''Return resident memory usage in bytes.
-        '''
-        return _VmB('VmRSS:') - since
-
-
-    def stacksize(since=0.0):
-        '''Return stack size in bytes.
-        '''
-        return _VmB('VmStk:') - since
-    ## end of http://code.activestate.com/recipes/286222/ }}}
     memory = linux_memory
+
 elif iswindows:
     import win32process
     import win32con
@@ -95,7 +191,7 @@ elif iswindows:
 
     def win_memory(since=0.0):
         info = meminfo(get_handle(os.getpid()))
-        return info['WorkingSetSize'] - since
+        return (info['WorkingSetSize']/1024.**2) - since
 
     memory = win_memory