iPad driver updates

src/calibre/devices/apple/driver.py

@@ -5,12 +5,13 @@ __copyright__ = '2010, Gregory Riker'
 __docformat__ = 'restructuredtext en'
 
 
-import cStringIO, os, re, shutil, sys, tempfile, time, zipfile
+import cStringIO, os, re, shutil, subprocess, sys, tempfile, time, zipfile
 
 from calibre.constants import DEBUG
 from calibre import fit_image
 from calibre.constants import isosx, iswindows
 from calibre.devices.interface import DevicePlugin
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.library.server.utils import strftime
 from calibre.ptempfile import PersistentTemporaryFile
@@ -106,6 +107,7 @@ class ITUNES(DevicePlugin):
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
     ejected = False
     iTunes= None
+    iTunes_media = None
     log = Log()
     path_template = 'iTunes/%s - %s.epub'
     problem_titles = []
@@ -138,12 +140,12 @@ class ITUNES(DevicePlugin):
                     self.log.info( "ITUNES.add_books_to_metadata()")
                     self._dump_update_list('add_books_to_metadata()')
                 for (j,p_book) in enumerate(self.update_list):
-                    self.log.info("ITUNES.add_books_to_metadata(): looking for %s" %
+                    self.log.info("ITUNES.add_books_to_metadata():\n looking for %s" %
                         str(p_book['lib_book'])[-9:])
                     for i,bl_book in enumerate(booklists[0]):
                         if bl_book.library_id == p_book['lib_book']:
                             booklists[0].pop(i)
-                            self.log.info("ITUNES.add_books_to_metadata(): removing %s %s" %
+                            self.log.info("ITUNES.add_books_to_metadata():\n removing %s %s" %
                                 (p_book['title'], str(p_book['lib_book'])[-9:]))
                             break
                     else:
@@ -180,8 +182,9 @@ class ITUNES(DevicePlugin):
                 self.log.info(" adding '%s' by '%s' to booklists[0]" %
                     (new_book.title, new_book.author))
             booklists[0].append(new_book)
-        if DEBUG:
+
-            self._dump_booklist(booklists[0],'after add_books_to_metadata()')
+#         if DEBUG:
+#             self._dump_booklist(booklists[0],'after add_books_to_metadata()')
 
     def books(self, oncard=None, end_session=True):
         """
@@ -595,7 +598,7 @@ class ITUNES(DevicePlugin):
                                 L{books}(oncard='cardb')).
         '''
         if DEBUG:
-            self.log.info("ITUNES.remove_books_from_metadata():")
+            self.log.info("ITUNES.remove_books_from_metadata()")
         for path in paths:
             if self.cached_books[path]['lib_book']:
                 # Remove from the booklist
@@ -605,15 +608,15 @@ class ITUNES(DevicePlugin):
                         booklists[0].pop(i)
                         break
                 else:
-                    self.log.error("ITUNES.remove_books_from_metadata(): '%s' not found in self.cached_book" % path)
+                    self.log.error(" '%s' not found in self.cached_book" % path)
 
                 # Remove from cached_books
                 self.cached_books.pop(path)
                 if DEBUG:
-                    self.log.info("ITUNES.remove_books_from_metadata(): Removing '%s' from self.cached_books" % path)
+                    self.log.info(" Removing '%s' from self.cached_books" % path)
-                    self._dump_cached_books('remove_books_from_metadata()')
+#                    self._dump_cached_books('remove_books_from_metadata()')
             else:
-                self.log.warning("ITUNES.remove_books_from_metadata(): skipping purchased book, can't remove via automation interface")
+                self.log.warning(" skipping purchased book, can't remove via automation interface")
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -657,54 +660,13 @@ class ITUNES(DevicePlugin):
                                 L{books}(oncard='cardb')).
         '''
         if DEBUG:
-            self.log.info("ITUNES:sync_booklists():")
+            self.log.info("ITUNES:sync_booklists()")
         if self.update_needed:
             if DEBUG:
                 self.log.info(' calling _update_device')
             self._update_device(msg=self.update_msg, wait=False)
-            self.update_needed = False
-
-        # Get actual size of updated books on device
-        if self.update_list:
-            if DEBUG:
-                self._dump_update_list(header='sync_booklists()')
-            if isosx:
-                for updated_book in self.update_list:
-                    size_on_device = self._get_device_book_size(updated_book['title'],
-                                                                updated_book['author'][0])
-                    if size_on_device:
-                        for book in booklists[0]:
-                            if book.title == updated_book['title'] and \
-                               book.author == updated_book['author']:
-                                break
-                        else:
-                            self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
-
-                    else:
-                        self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
-
-            elif iswindows:
-                try:
-                    pythoncom.CoInitialize()
-                    self.iTunes = win32com.client.Dispatch("iTunes.Application")
-
-                    for updated_book in self.update_list:
-                        size_on_device = self._get_device_book_size(updated_book['title'], updated_book['author'])
-                        if size_on_device:
-                            for book in booklists[0]:
-                                if book.title == updated_book['title'] and \
-                                   book.author[0] == updated_book['author']:
-                                    book.size = size_on_device
-                                    break
-                            else:
-                                self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
-
-                        else:
-                            self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
-                finally:
-                    pythoncom.CoUninitialize()
-
             self.update_list = []
+            self.update_needed = False
 
         # Inform user of any problem books
         if self.problem_titles:
@@ -763,22 +725,49 @@ class ITUNES(DevicePlugin):
                                      "Click 'Show Details' for a list.")
 
         if DEBUG:
-            self.log.info("ITUNES.upload_books():")
+            self.log.info("ITUNES.upload_books()")
             self._dump_files(files, header='upload_books()')
-            self._dump_cached_books('upload_books()')
+#            self._dump_cached_books('upload_books()')
             self._dump_update_list('upload_books()')
 
         if isosx:
             for (i,file) in enumerate(files):
+                # Delete existing from Library|Books
+                # Add to self.update_list for deletion from booklist[0] during add_books_to_metadata
+
+                '''
+                # ---------------------------
+                # PROVISIONAL
+                # Use the cover to find the database storage point of the epub
+                # Pass database copy to iTunes instead of the temporary file
+
+                if False:
+                    if DEBUG:
+                        self.log.info(" processing '%s'" % metadata[i].title)
+                        self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
+                        self.log.info(" cover: %s" % metadata[i].cover)
+
+                    calibre_database_item = False
+                    if metadata[i].cover:
+                        passed_file = file
+                        storage_path =  os.path.split(metadata[i].cover)[0]
+                        try:
+                            database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
+                            file = os.path.join(storage_path,database_epub)
+                            calibre_database_item = True
+                            self.log.info(" using database file: %s" % file)
+                        except:
+                            self.log.info(" could not find epub in %s" % storage_path)
+                    else:
+                        self.log.info(" no cover available, using temp file")
+                # ---------------------------
+                '''
+
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
-                # Delete existing from Library|Books, add to self.update_list
-                # for deletion from booklist[0] during add_books_to_metadata
                 if path in self.cached_books:
                     if DEBUG:
                         self.log.info(" adding '%s' by %s to self.update_list" %
                                       (self.cached_books[path]['title'],self.cached_books[path]['author']))
-
-                    # *** Second time a book is updated the author is a list ***
                     self.update_list.append(self.cached_books[path])
 
                     if DEBUG:
@@ -826,16 +815,17 @@ class ITUNES(DevicePlugin):
                 this_book.device_collections = []
                 this_book.library_id = added
                 this_book.path = path
-                this_book.size = added.size()   # Updated later from actual storage size
+                this_book.size = self._get_device_book_size(file, added.size())
                 this_book.thumbnail = thumb
                 this_book.iTunes_id = added
 
                 new_booklist.append(this_book)
 
-                # Flesh out the iTunes metadata
+                # Populate the iTunes metadata
-                added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
                 if metadata[i].comments:
                     added.comment.set(strip_tags.sub('',metadata[i].comments))
+                added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
+                added.enabled.set(True)
                 if metadata[i].rating:
                     added.rating.set(metadata[i].rating*10)
                 added.sort_artist.set(metadata[i].author_sort.title())
@@ -859,6 +849,7 @@ class ITUNES(DevicePlugin):
                 # Report progress
                 if self.report_progress is not None:
                     self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))
+
         elif iswindows:
             try:
                 pythoncom.CoInitialize()
@@ -886,18 +877,39 @@ class ITUNES(DevicePlugin):
                         if DEBUG:
                             self.log.error(" no Books playlist found")
 
-                #
-#                 lib = self.iTunes.sources.ItemByName('Library')
-#                 lib_playlists = [pl.Name for pl in lib.Playlists]
-#                 if not 'Books' in lib_playlists:
-#                     self.log.error(" no 'Books' playlist in Library")
-#                 library_books = lib.Playlists.ItemByName('Books')
-                #
-
                 for (i,file) in enumerate(files):
-                    path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     # Delete existing from Library|Books, add to self.update_list
                     # for deletion from booklist[0] during add_books_to_metadata
+
+                    '''
+                    # ---------------------------
+                    # PROVISIONAL
+                    # Use the cover to find the database storage point of the epub
+                    # Pass database copy to iTunes instead of the temporary file
+
+                    if False:
+                        if DEBUG:
+                            self.log.info(" processing '%s'" % metadata[i].title)
+                            self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
+                            self.log.info(" cover: %s" % metadata[i].cover)
+
+                        calibre_database_item = False
+                        if metadata[i].cover:
+                            passed_file = file
+                            storage_path =  os.path.split(metadata[i].cover)[0]
+                            try:
+                                database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
+                                file = os.path.join(storage_path,database_epub)
+                                calibre_database_item = True
+                                self.log.info(" using database file: %s" % file)
+                            except:
+                                self.log.info(" could not find epub in %s" % storage_path)
+                        else:
+                            self.log.info(" no cover available, using temp file")
+                    # ---------------------------
+                    '''
+
+                    path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     if path in self.cached_books:
                         self.update_list.append(self.cached_books[path])
 
@@ -995,9 +1007,10 @@ class ITUNES(DevicePlugin):
                     new_booklist.append(this_book)
 
                     # Flesh out the iTunes metadata
-                    added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
                     if metadata[i].comments:
                         added.Comment = (strip_tags.sub('',metadata[i].comments))
+                    added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
+                    added.Enabled = True
                     if metadata[i].rating:
                         added.AlbumRating = (metadata[i].rating*10)
                     added.SortArtist = (metadata[i].author_sort.title())
@@ -1234,34 +1247,20 @@ class ITUNES(DevicePlugin):
                 self.log.error(" error generating thumb for '%s'" % book.Name)
                 return None
 
-    def _get_device_book_size(self, title, author):
+    def _get_device_book_size(self, file, compressed_size):
         '''
-        Fetch the size of a book stored on the device
+        Calculate the exploded size of file
         '''
+        myZip = zipfile.ZipFile(file,'r')
+        myZipList = myZip.infolist()
+        exploded_file_size = 0
+        for file in myZipList:
+            exploded_file_size += file.file_size
         if DEBUG:
-            self.log.info("ITUNES._get_device_book_size():\n looking for title: '%s' author: '%s'" %
+            self.log.info("ITUNES._get_device_book_size()")
-                (title,author))
+            self.log.info(" %d items in archive" % len(myZipList))
-
+            self.log.info(" compressed: %d  exploded: %d" % (compressed_size, exploded_file_size))
-        device_books = self._get_device_books()
+        return exploded_file_size
-
-        if isosx:
-            for d_book in device_books:
-                if d_book.name() == title and d_book.artist() == author:
-                    if DEBUG:
-                        self.log.info(' found it')
-                    return d_book.size()
-            else:
-                self.log.error("ITUNES._get_device_book_size():"
-                               " could not find '%s' by '%s' in device_books" % (title,author))
-                return None
-        elif iswindows:
-            for d_book in device_books:
-                if d_book.Name == title and d_book.Artist == author:
-                    self.log.info(" found it")
-                    return d_book.Size
-            else:
-                self.log.error(" could not find '%s' by '%s' in device_books" % (title,author))
-                return None
 
     def _get_device_books(self):
         '''
@@ -1463,11 +1462,21 @@ class ITUNES(DevicePlugin):
                 self.iTunes = appscript.app('iTunes')
                 initial_status = 'already running'
 
+            # Read the current storage path for iTunes media
+            cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
+            proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
+            retcode = proc.wait()
+            media_dir = proc.communicate()[0].strip()
+            if os.path.exists(media_dir):
+                self.iTunes_media = media_dir
+            else:
+                self.log.error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
+
             if DEBUG:
-                self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
+                self.log.info(" [%s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.name(), self.iTunes.version(), initial_status,
                   self.version[0],self.version[1],self.version[2]))
-
+                self.log.info(" iTunes_media: %s" % self.iTunes_media)
         if iswindows:
             '''
             Launch iTunes if not already running
@@ -1479,40 +1488,64 @@ class ITUNES(DevicePlugin):
                 self.iTunes.Windows[0].Minimized = True
             initial_status = 'launched'
 
+            # Read the current storage path for iTunes media from the XML file
+            with open(self.iTunes.LibraryXMLPath, 'r') as xml:
+                soup = BeautifulSoup(xml.read().decode('utf-8'))
+                mf = soup.find('key',text="Music Folder").parent
+                string = mf.findNext('string').renderContents()
+                media_dir = string[len('file://localhost/'):].replace('%20',' ')
+                if os.path.exists(media_dir):
+                    self.iTunes_media = media_dir
+                else:
+                    self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath)
+                    self.log.error(" %s" % string.parent.prettify())
+
             if DEBUG:
                 self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
                  (self.iTunes.Windows[0].name, self.iTunes.Version, initial_status,
                   self.version[0],self.version[1],self.version[2]))
+                self.log.info(" iTunes_media: %s" % self.iTunes_media)
 
     def _remove_from_iTunes(self, cached_book):
         '''
         iTunes does not delete books from storage when removing from database
+        We only want to delete stored copies if the file is stored in iTunes
+        We don't want to delete files stored outside of iTunes
         '''
         if isosx:
             storage_path = os.path.split(cached_book['lib_book'].location().path)
-            title_storage_path = storage_path[0]
+            presumptive_path = os.path.join(self.iTunes_media,
-            if DEBUG:
+                                            'iTunes Music',
-                self.log.info("ITUNES._remove_from_iTunes():")
+                                            cached_book['author'][0],
-                self.log.info(" removing title_storage_path: %s" % title_storage_path)
+                                            cached_book['title'],
-            try:
+                                            storage_path[1])
-                shutil.rmtree(title_storage_path)
-            except:
-                self.log.info(" '%s' not empty" % title_storage_path)
 
-            # Clean up title/author directories
+            if os.path.exists(presumptive_path):
-            author_storage_path = os.path.split(title_storage_path)[0]
+                title_storage_path = storage_path[0]
-            self.log.info(" author_storage_path: %s" % author_storage_path)
-            author_files = os.listdir(author_storage_path)
-            if '.DS_Store' in author_files:
-                author_files.pop(author_files.index('.DS_Store'))
-            if not author_files:
-                shutil.rmtree(author_storage_path)
                 if DEBUG:
-                    self.log.info(" removing empty author_storage_path")
+                    self.log.info("ITUNES._remove_from_iTunes():")
+                    self.log.info(" removing title_storage_path: %s" % title_storage_path)
+                try:
+                    shutil.rmtree(title_storage_path)
+                except:
+                    self.log.info(" '%s' not empty" % title_storage_path)
+
+                # Clean up title/author directories
+                author_storage_path = os.path.split(title_storage_path)[0]
+                self.log.info(" author_storage_path: %s" % author_storage_path)
+                author_files = os.listdir(author_storage_path)
+                if '.DS_Store' in author_files:
+                    author_files.pop(author_files.index('.DS_Store'))
+                if not author_files:
+                    shutil.rmtree(author_storage_path)
+                    if DEBUG:
+                        self.log.info(" removing empty author_storage_path")
+                else:
+                    if DEBUG:
+                        self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
+                        self.log.info(" %s" % '\n'.join(author_files))
             else:
-                if DEBUG:
+                self.log.info(" '%s' not found in iTunes storage, no files deleted" % presumptive_path)
-                    self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
-                    self.log.info(" %s" % '\n'.join(author_files))
 
             self.iTunes.delete(cached_book['lib_book'])
 
@@ -1522,26 +1555,42 @@ class ITUNES(DevicePlugin):
             Windows stores the book under a common author directory, so we just delete the .epub
             '''
             if DEBUG:
-                self.log.info("ITUNES._remove_from_iTunes(): '%s'" % cached_book['title'])
+                self.log.info("ITUNES._remove_from_iTunes():\n '%s'" % cached_book['title'])
             book = self._find_library_book(cached_book)
             if book:
-                if DEBUG:
-                    self.log.info("ITUNES._remove_from_iTunes():\n deleting '%s' at %s" %
-                        (cached_book['title'], book.Location))
-                folder = os.path.split(book.Location)[0]
                 path = book.Location
+                storage_path = os.path.split(book.Location)
+                # This assumes that 'Books' will be the storage subdirectory in
+                # all versions of Windows.  The XML file doesn't offer any deeper information
+                # than the media directory.
+                presumptive_path = os.path.join(self.iTunes_media,
+                                                'Books',
+                                                cached_book['author'],
+                                                storage_path[1])
+
+                if os.path.exists(presumptive_path):
+                    if DEBUG:
+                        self.log.info("ITUNES._remove_from_iTunes():")
+                        self.log.info(" removing '%s' at %s" %
+                            (cached_book['title'], path))
+                    try:
+                        os.remove(path)
+                    except:
+                        self.log.warning(" could not find '%s' in iTunes storage" % path)
+                    try:
+                        os.rmdir(storage_path[0])
+                        self.log.info(" removed folder '%s'" % storage_path[0])
+                    except:
+                        self.log.info(" folder '%s' not found or not empty" % storage_path[0])
+
+                    # Delete from iTunes database
+                else:
+                    self.log.info(" '%s' not in iTunes storage, no files deleted" % presumptive_path)
+
                 book.Delete()
-                try:
+
-                    os.remove(path)
-                except:
-                    self.log.warning(" could not find '%s' in iTunes storage" % path)
-                try:
-                    os.rmdir(folder)
-                    self.log.info(" removed folder '%s'" % folder)
-                except:
-                    self.log.info(" folder '%s' not found or not empty" % folder)
             else:
-                self.log.warning(" could not find '%s' in iTunes storage" % cached_book['title'])
+                self.log.warning(" could not find '%s' in iTunes database" % cached_book['title'])
 
     def _update_device(self, msg='', wait=True):
         '''