From 29209c748a8b655569f05f691e1891f47d89159c Mon Sep 17 00:00:00 2001 From: GRiker Date: Sun, 13 Jun 2010 03:47:09 -0600 Subject: [PATCH 1/8] Gwr wip --- src/calibre/web/feeds/news.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index f54d5bde9d..21b2127f90 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -787,6 +787,7 @@ class BasicNewsRecipe(Recipe): } .summary_byline { + text-align:left; font-family:monospace; } From a726347e2b6439634f6dd08728e54122c863e68f Mon Sep 17 00:00:00 2001 From: GRiker Date: Tue, 15 Jun 2010 04:13:35 -0600 Subject: [PATCH 2/8] GwR wip --- src/calibre/devices/apple/driver.py | 367 +++++++++++++++++++--------- 1 file changed, 256 insertions(+), 111 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index ae440a359e..293872487e 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -5,19 +5,21 @@ __copyright__ = '2010, Gregory Riker' __docformat__ = 'restructuredtext en' -import cStringIO, ctypes, os, re, shutil, subprocess, sys, tempfile, time, zipfile +import cStringIO, ctypes, datetime, os, re, shutil, subprocess, sys, tempfile, time from calibre.constants import DEBUG from calibre import fit_image from calibre.constants import isosx, iswindows +from calibre.devices.errors import UserFeedback from calibre.devices.interface import DevicePlugin -from calibre.ebooks.BeautifulSoup import BeautifulSoup +from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag from calibre.ebooks.metadata import MetaInformation from calibre.library.server.utils import strftime +from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import Config, config_dir -from calibre.utils.date import parse_date +from calibre.utils.date import isoformat, now, parse_date, strptime from calibre.utils.logging import Log -from calibre.devices.errors import UserFeedback +from calibre.utils.zipfile import safe_replace, ZipFile from PIL import Image as PILImage @@ -31,6 +33,8 @@ if isosx: if iswindows: import pythoncom, win32com.client + from calibre.ebooks.BeautifulSoup import BeautifulSoup + class ITUNES(DevicePlugin): ''' @@ -633,7 +637,7 @@ class ITUNES(DevicePlugin): if not os.path.exists(archive_path): self.log.info(" creating zip archive") - zfw = zipfile.ZipFile(archive_path, mode='w') + zfw = ZipFile(archive_path, mode='w') zfw.writestr("iTunes Thumbs Archive",'') zfw.close() else: @@ -786,7 +790,7 @@ class ITUNES(DevicePlugin): for (i,file) in enumerate(files): path = self.path_template % (metadata[i].title, metadata[i].author[0]) self._remove_existing_copies(path,file,metadata[i]) - fpath = self._get_fpath(file) + fpath = self._get_fpath(file, touch=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) thumb = self._cover_to_thumb(path, metadata[i], lb_added, db_added) this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb) @@ -813,8 +817,16 @@ class ITUNES(DevicePlugin): for (i,file) in enumerate(files): path = self.path_template % (metadata[i].title, metadata[i].author[0]) self._remove_existing_copies(path,file,metadata[i]) - fpath = self._get_fpath(file) + fpath = self._get_fpath(file, touch=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) + + if self.manual_sync_mode and not db_added: + # Problem finding added book, probably title/author change needing to be written to metadata + self.problem_msg = ("Title and/or author metadata mismatch with uploaded books.\n" + "Convert epub - epub to update edited metadata before uploading.\n" + "Click 'Show Details...' for affected books.") + self.problem_titles.append("'%s' by %s" % (metadata[i].title, metadata[i].author[0])) + thumb = self._cover_to_thumb(path, metadata[i], lb_added, db_added) this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb) new_booklist.append(this_book) @@ -843,9 +855,11 @@ class ITUNES(DevicePlugin): return (new_booklist, [], []) + # Private methods def _add_device_book(self,fpath, metadata): ''' + assumes pythoncom wrapper for windows ''' self.log.info(" ITUNES._add_device_book()") if isosx: @@ -867,69 +881,69 @@ class ITUNES(DevicePlugin): elif iswindows: if 'iPod' in self.sources: - try: - pythoncom.CoInitialize() - connected_device = self.sources['iPod'] - device = self.iTunes.sources.ItemByName(connected_device) + connected_device = self.sources['iPod'] + device = self.iTunes.sources.ItemByName(connected_device) - added = None - for pl in device.Playlists: - if pl.Kind == self.PlaylistKind.index('User') and \ - pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): - break - else: - if DEBUG: - self.log.info(" no Books playlist found") + added = None + for pl in device.Playlists: + if pl.Kind == self.PlaylistKind.index('User') and \ + pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): + break + else: + if DEBUG: + self.log.info(" no Books playlist found") - # Add the passed book to the Device|Books playlist - if pl: - ''' - added = pl.AddFile(fpath) - if DEBUG: - self.log.info(" adding '%s' to device" % fpath) - ''' - file_s = ctypes.c_char_p(fpath) - FileArray = ctypes.c_char_p * 1 - fa = FileArray(file_s) - op_status = pl.AddFiles(fa) + # Add the passed book to the Device|Books playlist + if pl: + file_s = ctypes.c_char_p(fpath) + FileArray = ctypes.c_char_p * 1 + fa = FileArray(file_s) + op_status = pl.AddFiles(fa) + if DEBUG: + sys.stdout.write(" uploading '%s' to device ..." % metadata.title) + sys.stdout.flush() + + while op_status.InProgress: + time.sleep(0.5) if DEBUG: - sys.stdout.write(" uploading '%s' to device ..." % metadata.title) + sys.stdout.write('.') sys.stdout.flush() + if DEBUG: + sys.stdout.write("\n") + sys.stdout.flush() - while op_status.InProgress: + # This doesn't seem to work with device, just Library + if False: + if DEBUG: + sys.stdout.write(" waiting for handle to added '%s' ..." % metadata.title) + sys.stdout.flush() + while not op_status.Tracks: time.sleep(0.5) if DEBUG: sys.stdout.write('.') sys.stdout.flush() + if DEBUG: - sys.stdout.write("\n") - sys.stdout.flush() + print + added = op_status.Tracks[0] + else: + # This approach simply scans Library|Books for the book we just added - # This doesn't seem to work with device, just Library - if False: - if DEBUG: - sys.stdout.write(" waiting for handle to added '%s' ..." % metadata.title) - sys.stdout.flush() - while op_status.Tracks is None: - time.sleep(0.5) - if DEBUG: - sys.stdout.write('.') - sys.stdout.flush() - if DEBUG: - print - added = op_status.Tracks[0] - else: - # This approach simply scans Library|Books for the book we just added - added = self._find_device_book( - {'title': metadata.title, - 'author': metadata.author[0]}) - return added + # Try the calibre metadata first + db_added = self._find_device_book( + {'title': metadata.title, + 'author': metadata.authors[0], + 'source': 'calibre'}) - finally: - pythoncom.CoUninitialize() - - return added + # If that fails, try the epub metadata + if not db_added: + title, author = self._get_epub_metadata(fpath) + db_added = self._find_device_book( + {'title': title, + 'author': author, + 'source': 'epub'}) + return db_added def _add_library_book(self,file, metadata): ''' @@ -1061,7 +1075,7 @@ class ITUNES(DevicePlugin): if DEBUG: self.log.info( " refreshing cached thumb for '%s'" % metadata.title) archive_path = os.path.join(self.cache_dir, "thumbs.zip") - zfw = zipfile.ZipFile(archive_path, mode='a') + zfw = ZipFile(archive_path, mode='a') thumb_path = path.rpartition('.')[0] + '.jpg' zfw.writestr(thumb_path, thumb) zfw.close() @@ -1168,7 +1182,11 @@ class ITUNES(DevicePlugin): self.manual_sync_mode = True except: self.manual_sync_mode = False - self.log.info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) + if DEBUG: + self.log.info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) + else: + if DEBUG: + self.log.error(" no books on iDevice, can't determine manual sync mode") def _dump_booklist(self, booklist, header=None): ''' @@ -1291,24 +1309,24 @@ class ITUNES(DevicePlugin): ub['author'])) self.log.info() - def _find_device_book(self, cached_book): + def _find_device_book(self, search): ''' Windows-only method to get a handle to device book in the current pythoncom session ''' if iswindows: if DEBUG: self.log.info(" ITUNES._find_device_book()") - self.log.info(" looking for '%s' by %s" % (cached_book['title'], cached_book['author'])) + self.log.info(" searching for '%s' by %s (%s metadata)" % + (search['title'], search['author'], search['source'])) dev_books = self._get_device_books_playlist() attempts = 9 while attempts: - # Find book whose Artist field = cached_book['author'] - hits = dev_books.Search(cached_book['author'],self.SearchField.index('Artists')) + hits = dev_books.Search(search['author'],self.SearchField.index('Artists')) if hits: for hit in hits: self.log.info(" evaluating '%s' by %s" % (hit.Name, hit.Artist)) - if hit.Name == cached_book['title']: + if hit.Name == search['title']: self.log.info(" matched '%s' by %s" % (hit.Name, hit.Artist)) return hit attempts -= 1 @@ -1317,7 +1335,8 @@ class ITUNES(DevicePlugin): self.log.warning(" attempt #%d" % (10 - attempts)) if DEBUG: - self.log.error(" search for '%s' yielded no hits" % cached_book['title']) + self.log.error(" search for '%s' using %s metadata yielded no hits" % + (search['title'], search['source'])) return None def _find_library_book(self, cached_book): @@ -1382,11 +1401,11 @@ class ITUNES(DevicePlugin): thumb_path = book_path.rpartition('.')[0] + '.jpg' try: - zfr = zipfile.ZipFile(archive_path) + zfr = ZipFile(archive_path) thumb_data = zfr.read(thumb_path) zfr.close() except: - zfw = zipfile.ZipFile(archive_path, mode='a') + zfw = ZipFile(archive_path, mode='a') else: return thumb_data @@ -1445,7 +1464,7 @@ class ITUNES(DevicePlugin): ''' Calculate the exploded size of file ''' - myZip = zipfile.ZipFile(file,'r') + myZip = ZipFile(file,'r') myZipList = myZip.infolist() exploded_file_size = 0 for file in myZipList: @@ -1542,7 +1561,31 @@ class ITUNES(DevicePlugin): self.log.error(" no iPad|Books playlist found") return pl - def _get_fpath(self,file): + def _get_epub_metadata(self, fpath): + ''' + Return the original title and author from the .OPF file in the epub bundle + ''' + self.log.info(" ITUNES.__get_epub_metadata()") + title = None + author = None + zf = ZipFile(fpath,'r') + fnames = zf.namelist() + opf = [x for x in fnames if '.opf' in x][0] + if opf: + opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue() + soup = BeautifulSoup(opf_raw) + title = soup.find('dc:title').renderContents() + author = soup.find('dc:creator').renderContents() + if not title or not author: + if DEBUG: + self.log.error(" couldn't extract title/author from %s in %s" % (opf,fpath)) + self.log.error(" title: %s author: %s" % (title, author)) + else: + if DEBUG: + self.log.error(" can't find .opf in %s" % fpath) + return title, author + + def _get_fpath(self,file, touch=False): ''' If the database copy will be deleted after upload, we have to use file (the PersistentTemporaryFile), which will be around until @@ -1555,6 +1598,8 @@ class ITUNES(DevicePlugin): if not getattr(fpath, 'deleted_after_upload', False): if getattr(file, 'orig_file_path', None) is not None: fpath = file.orig_file_path + if touch: + self._touch_epub(fpath) elif getattr(file, 'name', None) is not None: fpath = file.name else: @@ -1958,6 +2003,58 @@ class ITUNES(DevicePlugin): book.Delete() + def _touch_epub(self, fpath): + ''' + Touch calibre:timestamp in OPF to force iBooks to recache + ''' + self.log.info(" ITUNES._touch_epub()") + + title = None + author = None + zf = ZipFile(fpath,'r') + fnames = zf.namelist() + opf = [x for x in fnames if '.opf' in x][0] + if opf: + opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue() + soup = BeautifulSoup(opf_raw) + md = soup.find('metadata') + ots = ts = md.find('meta',attrs={'name':'calibre:timestamp'}) + if ts: + # Touch existing calibre timestamp + timestamp = ts['content'] +# old_ts = datetime.datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f+00:00") +# new_ts = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, old_ts.minute, +# old_ts.second, old_ts.microsecond+1) +# ts['content'] = new_ts.strftime("%Y-%m-%dT%H:%M:%S.%f+00:00") + old_ts = strptime(timestamp,"%Y-%m-%dT%H:%M:%S.%f+00:00") + new_ts = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, + old_ts.minute, old_ts.second, old_ts.microsecond+1) + ts['content'] = new_ts.strftime("%Y-%m-%dT%H:%M:%S.%f+00:00") + if DEBUG: + self.log.info(" touching existing calibre:timestamp in %s" % opf) + self.log.info(" %s" % ots) + self.log.info(" %s" % ts) + else: + # Create new calibre timestamp + if True: + print "existing metadata:\n%s" % md.prettify() + else: + ts = Tag(soup,'meta') + ts['name'] = 'calibre:timestamp' + ts['content'] = isoformat(now()) + md.insert(len(md),ts) + if DEBUG: + self.log.info(" adding calibre:timestamp to %s" % opf) + self.log.info(" %s" % ts) + zfo = open(fpath,'r+b') + safe_replace(zfo, opf, cStringIO.StringIO(soup)) + + else: + if DEBUG: + self.log.error(" can't find .opf in %s" % fpath) + + + def _update_device(self, msg='', wait=True): ''' Trigger a sync, wait for completion @@ -2014,6 +2111,24 @@ class ITUNES(DevicePlugin): strip_tags = re.compile(r'<[^<]*?/?>') if isosx: + if lb_added: + lb_added.album.set(metadata.title) + lb_added.artist.set(metadata.authors[0]) + lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) + lb_added.enabled.set(True) + lb_added.name.set(metadata.title) + lb_added.sort_artist.set(metadata.author_sort.title()) + lb_added.sort_name.set(this_book.title_sorter) + + if db_added: + db_added.album.set(metadata.title) + db_added.artist.set(metadata.authors[0]) + db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) + db_added.enabled.set(True) + db_added.name.set(metadata.title) + db_added.sort_artist.set(metadata.author_sort.title()) + db_added.sort_name.set(this_book.title_sorter) + if metadata.comments: if lb_added: lb_added.comment.set(strip_tags.sub('',metadata.comments)) @@ -2030,31 +2145,46 @@ class ITUNES(DevicePlugin): except: pass - if lb_added: - lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) - lb_added.enabled.set(True) - lb_added.sort_artist.set(metadata.author_sort.title()) - lb_added.sort_name.set(this_book.title_sorter) - - if db_added: - db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) - db_added.enabled.set(True) - db_added.sort_artist.set(metadata.author_sort.title()) - db_added.sort_name.set(this_book.title_sorter) - - # Set genre from metadata - # iTunes grabs the first dc:subject from the opf metadata, - # But we can manually override with first tag starting with alpha - for tag in metadata.tags: - if self._is_alpha(tag[0]): - if lb_added: - lb_added.genre.set(tag) - if db_added: - db_added.genre.set(tag) - break - + # Set genre from series if available, else first alpha tag + # Otherwise iTunes grabs the first dc:subject from the opf metadata, + if metadata.series: + if lb_added: + lb_added.genre.set(metadata.series) + lb_added.episode_ID.set(metadata.series) + lb_added.episode_number.set(metadata.series_index) + if db_added: + db_added.genre.set(metadata.series) + db_added.episode_ID.set(metadata.series) + db_added.episode_number.set(metadata.series_index) + else: + for tag in metadata.tags: + if self._is_alpha(tag[0]): + if lb_added: + lb_added.genre.set(tag) + if db_added: + db_added.genre.set(tag) + break elif iswindows: + if lb_added: + lb_added.Album = metadata.title + lb_added.Artist = metadata.authors[0] + lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) + lb_added.Enabled = True + lb_added.Name = metadata.title + lb_added.SortArtist = (metadata.author_sort.title()) + lb_added.SortName = (this_book.title_sorter) + + if db_added: + # Album, Artist and Name are changed in _add_device_book() + db_added.Album = metadata.title + db_added.Artist = metadata.authors[0] + db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) + db_added.Enabled = True + db_added.Name = metadata.title + db_added.SortArtist = (metadata.author_sort.title()) + db_added.SortName = (this_book.title_sorter) + if metadata.comments: if lb_added: lb_added.Comment = (strip_tags.sub('',metadata.comments)) @@ -2071,27 +2201,42 @@ class ITUNES(DevicePlugin): except: pass - if lb_added: - lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) - lb_added.Enabled = True - lb_added.SortArtist = (metadata.author_sort.title()) - lb_added.SortName = (this_book.title_sorter) + # Set Category from first alpha tag, overwrite with series if available + # Otherwise iBooks uses first from opf + # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12) - if db_added: - db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) - db_added.SortArtist = (metadata.author_sort.title()) - db_added.SortName = (this_book.title_sorter) + if metadata.tags: + if DEBUG: + self.log.info(" setting Category from metadata.tags") + for tag in metadata.tags: + if self._is_alpha(tag[0]): + if lb_added: + lb_added.Category = tag + if db_added: + db_added.Category = tag + break - # Set genre from metadata - # iTunes grabs the first dc:subject from the opf metadata, - # But we can manually override with first tag starting with alpha - for tag in metadata.tags: - if self._is_alpha(tag[0]): - if lb_added: - lb_added.Category = (tag) - if db_added: - db_added.Category = (tag) - break + if metadata.series: + if DEBUG: + self.log.info(" setting Category from metadata.series") + if lb_added: + if DEBUG: + self.log.info(" setting lb_added from metadata.series") + lb_added.Category = metadata.series + lb_added.EpisodeID = metadata.series + try: + lb_added.EpisodeNumber = metadata.series_index + except: + pass + if db_added: + if DEBUG: + self.log.info(" setting db_added from metadata.series") + db_added.Category = metadata.series + db_added.EpisodeID = metadata.series + try: + db_added.EpisodeNumber = metadata.series_index + except: + pass class BookList(list): ''' From 214a35e174fce7b07de1086e04dbd89d4cc1792e Mon Sep 17 00:00:00 2001 From: GRiker Date: Tue, 15 Jun 2010 06:59:24 -0600 Subject: [PATCH 3/8] GwR wip --- resources/tracer.epub | Bin 0 -> 1806 bytes src/calibre/devices/apple/driver.py | 62 +++++++++++++++------------- 2 files changed, 33 insertions(+), 29 deletions(-) create mode 100644 resources/tracer.epub diff --git a/resources/tracer.epub b/resources/tracer.epub new file mode 100644 index 0000000000000000000000000000000000000000..225da3d46e70b28942cc725246c7a4a10d84ff83 GIT binary patch literal 1806 zcmZ`(3pkT|9Nx;JNv4)qHFBs>#?isi+%IJgrbV4fF0)wdLJK)g?i#skol=Qt)r^ig zQ<-vUA#%TuFv_K{$n|Jv=W}|xJkQbdJ-_eyf6w>5@ALhB@B4eP_G_h}pfw;6Nccmb zF~u`~FB~|plnhWLB8fmdPbGL#ss2Q~CyhuUqX^VQmd6 z3m~R}QUuDu=7_~UJ4ag-ooe+}0OL8T&N*87=iju(QwMeswO|nrwkaAV# z3AtT8sd=}bHWA%z?kQWntKazVyMfI3ElG?Oh83 zDFAIb*;rz6|Mf&5({w1*)3NCmDQ&v2Sdnzx79 z{DZ1?B+G;H^g_>x5cCyHY0h64ym_q9diJ)ls9YZBwmM=?+cHO8=}7dNftDv+Crs-F zt3wCCfiKeBo0H(OuWTOZ{|a4)V799qbL&iO6zrccb8Xz*xa3$XS`1I(DaQ)-XBLnO6h9IWAv#>^@h7q&=^j*s# zQ_WL%csVgotmXOeD##^)AYF{vZ&X_hZB1EX?ZXG_l zv6bzu<`g!LU-zcJY&#yxrS!bv5qmz)E;p!rRkp>e80{Nx0+j%Y-UHM{d{LH02%({D z0|C!Ec+zNqX?kxfZbWFKnevXwivzT&sw{cG%IY%F1nv4^w&YB zA8n3HyawU2B=5>G|1@&PvRsS>{m})F z3e4@PO#w=A9>*ZnPjkbnqMto$2G^MEx#v+e+%{~5d-!`CD~3@6o6$4wu)9Axk$xJxYuOg2_4ioKGxSlm?CH-f+$jGBO{`hj2cprXX!DE#Qh zoh*Y%2?qOGcif03)jaB17)$*eG}%GCm|J-QB557yf);3EuVeJ%5T>sXDH! z(C!OYyeVJ3i0y>hmjsPZNr`8Jw&A?grwhUxMSW*8XFia!DtR2j8CthIx?CQ!+m;!+ zWnMdUTjJlN8A!9teaz{zwDrb4mb0{gV10;z`Hcaz4$Nno{n%(;EL1fwyHSk%^CxFL z+dhATEI4sVsLZQs$|!Ep?6jouvo+9m>5nzJ7wdED-<#=jM1$1m-99!3TEulssyw4R8wQ~`O5Z?(Eo@u0{4kQW6U}eu` zxl^%ran_3QjLb|-8*)g+I!>=S&o&HR>UHFJLE d$%|Bz`oi Date: Tue, 15 Jun 2010 07:36:18 -0600 Subject: [PATCH 4/8] GwR wip --- resources/tracer.epub | Bin 1806 -> 2303 bytes src/calibre/devices/apple/driver.py | 21 ++++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/resources/tracer.epub b/resources/tracer.epub index 225da3d46e70b28942cc725246c7a4a10d84ff83..0c31dbcbbbe8ed5dc2371a3e0852e93a9de7f71a 100644 GIT binary patch delta 532 zcmeC<`!Bd*A8S1`iwFY)0|&!xi}N=8QZWv-j0_Ctm>3v9qM0G3d8x&|sU?Xii6x18 z1v#0;C8a^9^9~sZ*i2tvP`oZ#y(J-lgDFT$%Oy~nFZ61&Cr`wa(A$}(?q8qMuF*ex zueI@R<89afdJ2Cy+w!FHnxT%cbVF*O+~=H|hG`E4gciT9zrN@9gH(Zi{_>6!3sxQ| zxhCJwcA$adYnM_Y2Wuk3vd0&rIkvDl^$RmgXfN$Da|#WeS<-E^*yb*i>a4Ck2~N4k zqIj~JBv;?KkX-ia|Fw%(XSknQf44Gw+sBmwQbr3kKVSH^?sO36-nH@sG|bCb-8Qm;DyI`l|AcvRQqx>UqMio%hR6 zSa3ZfN}c)pq^B{rBMcc&_ta12`Sio-%gQQNtCk%-Gg#ykPaY_q$+((v!|l8YX4@Kj zb^lZ@taY83-laS9gXi7C*qlg~>v!4mZkj2|S($9jEs;_V*V!wUq>*vth(TqWtCRW3 z-W%(UQ}jgc?7eE4=(Dtn!C7zH+Sxi9qM=J|Ch@zyQlG~+&F0JZ{6Dw!H}7G!V+>=2 k#t`;+14bPKFdBhi4=eUq4De=U1Ie=iVI5FuHamz102G Date: Wed, 16 Jun 2010 14:29:20 -0600 Subject: [PATCH 5/8] GwR revisions --- src/calibre/devices/apple/driver.py | 186 ++++++++++-------------- src/calibre/ebooks/metadata/__init__.py | 2 +- src/calibre/ebooks/metadata/opf2.py | 2 +- src/calibre/web/feeds/news.py | 6 - 4 files changed, 81 insertions(+), 115 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index f5b8b46733..35b351ee37 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -237,8 +237,6 @@ class ITUNES(DevicePlugin): (new_book.title, new_book.author)) booklists[0].append(new_book) - self.update_list = [] - if DEBUG: self._dump_booklist(booklists[0],header='after',indent=2) self._dump_cached_books(header='after',indent=2) @@ -414,7 +412,7 @@ class ITUNES(DevicePlugin): self.ejected = True return False - self._discover_manual_sync_mode() + self._discover_manual_sync_mode(wait = 2 if self.initial_status == 'launched' else 0) return True def can_handle_windows(self, device_id, debug=False): @@ -676,25 +674,18 @@ class ITUNES(DevicePlugin): self.log.info("ITUNES.remove_books_from_metadata()") for path in paths: self._dump_cached_book(self.cached_books[path], indent=2) - if self.cached_books[path]['lib_book']: - # Remove from the booklist - for i,book in enumerate(booklists[0]): - print "book.uuid: %s" % book.uuid - print "self.cached_books[path]['uuid']: %s" % self.cached_books[path]['uuid'] - if book.uuid == self.cached_books[path]['uuid']: - booklists[0].pop(i) - break - else: - self.log.error(" '%s' not found in self.cached_book" % path) - # Remove from cached_books - self.cached_books.pop(path) - if DEBUG: - self.log.info(" removing '%s' from self.cached_books" % path) - self._dump_cached_books(header='remove_books_from_metadata()',indent=2) - self._dump_booklist(booklists[0], header='remove_books_from_metadata()',indent=2) - else: - self.log.warning(" skipping purchased book, can't remove via automation interface") + # Purge the booklist, self.cached_books + for i,bl_book in enumerate(booklists[0]): + if bl_book.uuid == self.cached_books[path]['uuid']: + # Remove from booklists[0] + booklists[0].pop(i) + + for cb in self.cached_books: + if self.cached_books[cb]['uuid'] == self.cached_books[path]['uuid']: + self.cached_books.pop(cb) + break + break def reset(self, key='-1', log_packets=False, report_progress=None, detected_device=None) : @@ -749,6 +740,7 @@ class ITUNES(DevicePlugin): details='\n'.join(self.problem_titles), level=UserFeedback.WARN) self.problem_titles = [] self.problem_msg = None + self.update_list = [] def total_space(self, end_session=True): """ @@ -1152,6 +1144,8 @@ class ITUNES(DevicePlugin): ''' if DEBUG: self.log.info(" ITUNES._discover_manual_sync_mode()") + if wait: + time.sleep(wait) if isosx: connected_device = self.sources['iPod'] dev_books = None @@ -1165,8 +1159,8 @@ class ITUNES(DevicePlugin): if len(dev_books): first_book = dev_books[0] - #if DEBUG: - #self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) + if DEBUG: + self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) try: first_book.bpm.set(0) self.manual_sync_mode = True @@ -1184,8 +1178,6 @@ class ITUNES(DevicePlugin): self.manual_sync_mode = False elif iswindows: - if wait: - time.sleep(wait) connected_device = self.sources['iPod'] device = self.iTunes.sources.ItemByName(connected_device) @@ -1294,18 +1286,35 @@ class ITUNES(DevicePlugin): self.log.info() - def _dump_hex(self, src, length=16): + def _dump_epub_metadata(self, fpath): ''' ''' - FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) - N=0; result='' - while src: - s,src = src[:length],src[length:] - hexa = ' '.join(["%02X"%ord(x) for x in s]) - s = s.translate(FILTER) - result += "%04X %-*s %s\n" % (N, length*3, hexa, s) - N+=length - print result + self.log.info(" ITUNES.__get_epub_metadata()") + title = None + author = None + timestamp = None + zf = ZipFile(fpath,'r') + fnames = zf.namelist() + opf = [x for x in fnames if '.opf' in x][0] + if opf: + opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue() + soup = BeautifulSoup(opf_raw) + title = soup.find('dc:title').renderContents() + author = soup.find('dc:creator').renderContents() + ts = soup.find('meta',attrs={'name':'calibre:timestamp'}) + if ts: + # Touch existing calibre timestamp + timestamp = ts['content'] + + if not title or not author: + if DEBUG: + self.log.error(" couldn't extract title/author from %s in %s" % (opf,fpath)) + self.log.error(" title: %s author: %s timestamp: %s" % (title, author, timestamp)) + else: + if DEBUG: + self.log.error(" can't find .opf in %s" % fpath) + zf.close() + return (title, author, timestamp) def _dump_files(self, files, header=None,indent=0): if header: @@ -1319,6 +1328,19 @@ class ITUNES(DevicePlugin): self.log.info(" %s%s" % (' '*indent,file.name)) self.log.info() + def _dump_hex(self, src, length=16): + ''' + ''' + FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) + N=0; result='' + while src: + s,src = src[:length],src[length:] + hexa = ' '.join(["%02X"%ord(x) for x in s]) + s = s.translate(FILTER) + result += "%04X %-*s %s\n" % (N, length*3, hexa, s) + N+=length + print result + def _dump_library_books(self, library_books): ''' ''' @@ -1615,30 +1637,6 @@ class ITUNES(DevicePlugin): self.log.error(" no iPad|Books playlist found") return pl - def _get_epub_metadata(self, fpath): - ''' - Return the original title and author from the .OPF file in the epub bundle - ''' - self.log.info(" ITUNES.__get_epub_metadata()") - title = None - author = None - zf = ZipFile(fpath,'r') - fnames = zf.namelist() - opf = [x for x in fnames if '.opf' in x][0] - if opf: - opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue() - soup = BeautifulSoup(opf_raw) - title = soup.find('dc:title').renderContents() - author = soup.find('dc:creator').renderContents() - if not title or not author: - if DEBUG: - self.log.error(" couldn't extract title/author from %s in %s" % (opf,fpath)) - self.log.error(" title: %s author: %s" % (title, author)) - else: - if DEBUG: - self.log.error(" can't find .opf in %s" % fpath) - return title, author - def _get_fpath(self,file, metadata, update_md=False): ''' If the database copy will be deleted after upload, we have to @@ -1652,13 +1650,17 @@ class ITUNES(DevicePlugin): if not getattr(fpath, 'deleted_after_upload', False): if getattr(file, 'orig_file_path', None) is not None: fpath = file.orig_file_path - if update_md: - self._update_epub_metadata(fpath, metadata) elif getattr(file, 'name', None) is not None: fpath = file.name else: if DEBUG: self.log.info(" file will be deleted after upload") + if update_md: + if DEBUG: + self.log.info(" metadata before rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) + self._update_epub_metadata(fpath, metadata) + if DEBUG: + self.log.info(" metadata after rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) return fpath def _get_library_books(self): @@ -1843,10 +1845,10 @@ class ITUNES(DevicePlugin): self.log.info( "ITUNES:open(): Launching iTunes" ) self.iTunes = iTunes= appscript.app('iTunes', hide=True) iTunes.run() - initial_status = 'launched' + self.initial_status = 'launched' else: self.iTunes = appscript.app('iTunes') - initial_status = 'already running' + self.initial_status = 'already running' # Read the current storage path for iTunes media cmd = "defaults read com.apple.itunes NSNavLastRootDirectory" @@ -1860,7 +1862,7 @@ class ITUNES(DevicePlugin): self.log.error(" media_dir: %s" % media_dir) if DEBUG: self.log.info(" [OSX %s - %s (%s), driver version %d.%d.%d]" % - (self.iTunes.name(), self.iTunes.version(), initial_status, + (self.iTunes.name(), self.iTunes.version(), self.initial_status, self.version[0],self.version[1],self.version[2])) self.log.info(" iTunes_media: %s" % self.iTunes_media) if iswindows: @@ -1872,7 +1874,7 @@ class ITUNES(DevicePlugin): self.iTunes = win32com.client.Dispatch("iTunes.Application") if not DEBUG: self.iTunes.Windows[0].Minimized = True - initial_status = 'launched' + self.initial_status = 'launched' # Read the current storage path for iTunes media from the XML file with open(self.iTunes.LibraryXMLPath, 'r') as xml: @@ -1889,7 +1891,7 @@ class ITUNES(DevicePlugin): if DEBUG: self.log.info(" [Windows %s - %s (%s), driver version %d.%d.%d]" % - (self.iTunes.Windows[0].name, self.iTunes.Version, initial_status, + (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status, self.version[0],self.version[1],self.version[2])) self.log.info(" iTunes_media: %s" % self.iTunes_media) @@ -2061,61 +2063,31 @@ class ITUNES(DevicePlugin): def _update_epub_metadata(self, fpath, metadata): ''' - Refresh metadata in database epub to force iBooks to recache ''' self.log.info(" ITUNES._update_epub_metadata()") # Refresh epub metadata with open(fpath,'r+b') as zfo: + + # Touch the timestamp to force a recache if metadata.timestamp: - #old_ts = strptime(metadata.timestamp,"%Y-%m-%dT%H:%M:%S.%f+00:00") old_ts = metadata.timestamp metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, - old_ts.minute, old_ts.second, old_ts.microsecond+1) + old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) else: metadata.timestamp = isoformat(now()) - if iswindows: - # This hack compensates for the Windows automation interface not allowing - # us to update Category after the fact. By removing the tags, we should be - # able to set the Category after the upload. - if metadata.series: - metadata.tags = None + # Tweak the author if 'News' in tags for friendlier display in iBooks + if _('News') in metadata.tags: + if metadata.title.find('[') > 0: + metadata.title = metadata.title[:metadata.title.find('[')-1] + date_as_author = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) + metadata.author = metadata.authors = [date_as_author] + sort_author = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', metadata.title).rstrip() + metadata.author_sort = '%s %s' % (sort_author, strftime('%Y-%m-%d')) - if DEBUG: - self.log.info(" updating to '%s' by %s" % (metadata.title, metadata.authors[0])) set_metadata(zfo,metadata) - if False: - # This code operates directly on the OPF file, obsolete since we're rewriting md - zf = ZipFile(fpath,'r') - fnames = zf.namelist() - opf = [x for x in fnames if '.opf' in x][0] - if opf: - opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue() - soup = BeautifulSoup(opf_raw) - md = soup.find('metadata') - ts = md.find('meta',attrs={'name':'calibre:timestamp'}) - if ts: - # Touch existing calibre timestamp - timestamp = ts['content'] - old_ts = strptime(timestamp,"%Y-%m-%dT%H:%M:%S.%f+00:00") - new_ts = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, - old_ts.minute, old_ts.second, old_ts.microsecond+1) - ts['content'] = new_ts.strftime("%Y-%m-%dT%H:%M:%S.%f+00:00") - else: - # Create new calibre timestamp - ts = Tag(soup,'meta') - ts['name'] = 'calibre:timestamp' - ts['content'] = isoformat(now()) - md.insert(len(md),ts) - zfo = open(fpath,'r+b') - safe_replace(zfo, opf, cStringIO.StringIO(soup.renderContents())) - - else: - if DEBUG: - self.log.error(" can't find .opf in %s" % fpath) - def _update_device(self, msg='', wait=True): ''' Trigger a sync, wait for completion diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 8caca1f261..6e64055cd7 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -256,7 +256,7 @@ class MetaInformation(object): setattr(self, x, getattr(mi, x, None)) def print_all_attributes(self): - for x in ('author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher', + for x in ('title','author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher', 'series', 'series_index', 'tags', 'rating', 'isbn', 'language', 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 3367ab14f6..88a6633ae7 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -918,7 +918,7 @@ class OPF(object): for attr in ('title', 'authors', 'author_sort', 'title_sort', 'publisher', 'series', 'series_index', 'rating', 'isbn', 'language', 'tags', 'category', 'comments', - 'pubdate'): + 'pubdate','timestamp'): val = getattr(mi, attr, None) if val is not None and val != [] and val != (None, None): setattr(self, attr, val) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index 58a943c62d..73e0fae8e8 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -1140,12 +1140,6 @@ class BasicNewsRecipe(Recipe): mi = MetaInformation(self.short_title() + strftime(self.timefmt), [__appname__]) mi.publisher = __appname__ mi.author_sort = __appname__ - if self.output_profile.name == 'iPad': - date_as_author = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) - mi = MetaInformation(self.short_title(), [date_as_author]) - mi.publisher = __appname__ - sort_author = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip() - mi.author_sort = '%s %s' % (sort_author, strftime('%Y-%m-%d')) mi.publication_type = 'periodical:'+self.publication_type mi.timestamp = nowf() mi.comments = self.description From 1d28a78242015fb1baf62019036502bc13a43034 Mon Sep 17 00:00:00 2001 From: GRiker Date: Thu, 17 Jun 2010 10:40:25 -0600 Subject: [PATCH 6/8] GwR revisions --- src/calibre/devices/apple/driver.py | 300 +++++++++++++++++----------- 1 file changed, 181 insertions(+), 119 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 35b351ee37..3c4f60edb7 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -18,7 +18,7 @@ from calibre.ebooks.metadata.epub import set_metadata from calibre.library.server.utils import strftime from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import Config, config_dir -from calibre.utils.date import isoformat, now, parse_date, strptime +from calibre.utils.date import fromtimestamp, isoformat, now, parse_date, strptime from calibre.utils.logging import Log from calibre.utils.zipfile import safe_replace, ZipFile @@ -180,14 +180,14 @@ class ITUNES(DevicePlugin): # Delete any obsolete copies of the book from the booklist if self.update_list: - if DEBUG: + if True: self.log.info("ITUNES.add_books_to_metadata()") - self._dump_booklist(booklists[0], header='before',indent=2) - self._dump_update_list(header='before',indent=2) - self._dump_cached_books(header='before',indent=2) + #self._dump_booklist(booklists[0], header='before',indent=2) + #self._dump_update_list(header='before',indent=2) + #self._dump_cached_books(header='before',indent=2) for (j,p_book) in enumerate(self.update_list): - if DEBUG: + if False: if isosx: self.log.info(" looking for %s" % str(p_book['lib_book'])[-9:]) @@ -200,12 +200,12 @@ class ITUNES(DevicePlugin): if bl_book.uuid == p_book['uuid']: # Remove from booklists[0] booklists[0].pop(i) - if DEBUG: + if False: if isosx: - self.log.info(" removing %s %s from booklists[0]" % + self.log.info(" removing old %s %s from booklists[0]" % (p_book['title'], str(p_book['lib_book'])[-9:])) elif iswindows: - self.log.info(" removing '%s' from booklists[0]" % + self.log.info(" removing old '%s' from booklists[0]" % (p_book['title'])) # If >1 matching uuid, remove old title @@ -232,12 +232,12 @@ class ITUNES(DevicePlugin): # Add new books to booklists[0] for new_book in locations[0]: - if DEBUG: + if False: self.log.info(" adding '%s' by '%s' to booklists[0]" % (new_book.title, new_book.author)) booklists[0].append(new_book) - if DEBUG: + if False: self._dump_booklist(booklists[0],header='after',indent=2) self._dump_cached_books(header='after',indent=2) @@ -643,6 +643,8 @@ class ITUNES(DevicePlugin): Note that most of the initialization is necessarily performed in can_handle(), as we need to talk to iTunes to discover if there's a connected iPod ''' + if DEBUG: + self.log.info("ITUNES.open()") # Confirm/create thumbs archive archive_path = os.path.join(self.cache_dir, "thumbs.zip") @@ -790,7 +792,7 @@ class ITUNES(DevicePlugin): self.problem_msg = _("Some cover art could not be converted.\n" "Click 'Show Details' for a list.") - if DEBUG: + if False: self.log.info("ITUNES.upload_books()") self._dump_files(files, header='upload_books()',indent=2) self._dump_update_list(header='upload_books()',indent=2) @@ -862,7 +864,7 @@ class ITUNES(DevicePlugin): self.update_needed = True self.update_msg = "Added books to device" - if DEBUG: + if False: self._dump_booklist(new_booklist,header="after upload_books()",indent=2) self._dump_cached_books(header="after upload_books()",indent=2) return (new_booklist, [], []) @@ -887,8 +889,8 @@ class ITUNES(DevicePlugin): # Add the passed book to the Device|Books playlist added = pl.add(appscript.mactypes.File(fpath),to=pl) - if DEBUG: - self.log.info(" adding '%s' to device" % fpath) + if False: + self.log.info(" '%s' added to Device|Books" % metadata.title) return added elif iswindows: @@ -913,7 +915,7 @@ class ITUNES(DevicePlugin): op_status = pl.AddFiles(fa) if DEBUG: - sys.stdout.write(" uploading '%s' to device ..." % metadata.title) + sys.stdout.write(" uploading '%s' to Device|Books ..." % metadata.title) sys.stdout.flush() while op_status.InProgress: @@ -982,7 +984,7 @@ class ITUNES(DevicePlugin): sys.stdout.write("\n") sys.stdout.flush() - if True: + if False: if DEBUG: sys.stdout.write(" waiting for handle to added '%s' ..." % metadata.title) sys.stdout.flush() @@ -997,8 +999,9 @@ class ITUNES(DevicePlugin): else: # This approach simply scans Library|Books for the book we just added added = self._find_library_book( - {'title': metadata.title, - 'author': metadata.author[0]}) + { 'title': metadata.title, + 'author': metadata.author[0], + 'uuid': metadata.uuid}) return added def _add_new_copy(self, fpath, metadata): @@ -1012,12 +1015,11 @@ class ITUNES(DevicePlugin): if self.manual_sync_mode: db_added = self._add_device_book(fpath, metadata) - if DEBUG: - self.log.info(" file uploaded to Device|Books") if not getattr(fpath, 'deleted_after_upload', False): lb_added = self._add_library_book(fpath, metadata) - if DEBUG: - self.log.info(" file added to Library|Books for iTunes:iBooks tracking") + if lb_added: + if DEBUG: + self.log.info(" file added to Library|Books for iTunes<->iBooks tracking") else: lb_added = self._add_library_book(fpath, metadata) if DEBUG: @@ -1044,7 +1046,7 @@ class ITUNES(DevicePlugin): db_added.artworks[1].data_.set(cover_data.read()) except: if DEBUG: - self.log.warning(" iTunes automation interface generated an error" + self.log.warning(" iTunes automation interface reported an error" " when adding artwork to '%s'" % metadata.title) #import traceback #traceback.print_exc() @@ -1136,11 +1138,27 @@ class ITUNES(DevicePlugin): return this_book + def _delete_iTunesMetadata_plist(self,fpath): + ''' + Delete the plist file from the file to force recache + ''' + zf = ZipFile(fpath,'a') + fnames = zf.namelist() + pl_name = 'iTunesMetadata.plist' + try: + plist = [x for x in fnames if pl_name in x][0] + except: + plist = None + if plist: + if DEBUG: + self.log.info(" deleting %s from %s" % (pl_name,fpath)) + zf.delete(pl_name) + zf.close() + def _discover_manual_sync_mode(self, wait=0): ''' Assumes pythoncom for windows wait is passed when launching iTunes, as it seems to need a moment to come to its senses - ''' if DEBUG: self.log.info(" ITUNES._discover_manual_sync_mode()") @@ -1159,7 +1177,7 @@ class ITUNES(DevicePlugin): if len(dev_books): first_book = dev_books[0] - if DEBUG: + if False: self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) try: first_book.bpm.set(0) @@ -1383,28 +1401,20 @@ class ITUNES(DevicePlugin): (search['uuid'], search['title'], search['author'])) attempts = 9 while attempts: + # Try by uuid hits = dev_books.Search(search['uuid'],self.SearchField.index('Albums')) if hits: hit = hits[0] self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Album)) return hit - attempts -= 1 - time.sleep(0.5) - if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) - # Try again with title - if DEBUG: - self.log.info(" ITUNES._find_device_book(author)") - self.log.info(" searching for '%s' by %s" % - (search['title'], search['author'])) - attempts = 9 - while attempts: + # Try by author hits = dev_books.Search(search['author'],self.SearchField.index('Artists')) if hits: hit = hits[0] self.log.info(" found '%s' by %s" % (hit.Name, hit.Artist)) return hit + attempts -= 1 time.sleep(0.5) if DEBUG: @@ -1421,8 +1431,12 @@ class ITUNES(DevicePlugin): if iswindows: if DEBUG: self.log.info(" ITUNES._find_library_book()") - self.log.info(" looking for '%s' by %s (%s)" % + if 'uuid' in cached_book: + self.log.info(" looking for '%s' by %s (%s)" % (cached_book['title'], cached_book['author'], cached_book['uuid'])) + else: + self.log.info(" looking for '%s' by %s" % + (cached_book['title'], cached_book['author'])) for source in self.iTunes.sources: if source.Kind == self.Sources.index('Library'): @@ -1450,16 +1464,26 @@ class ITUNES(DevicePlugin): attempts = 9 while attempts: # Find book whose Album field = cached_book['uuid'] - hits = lib_books.Search(cached_book['uuid'],self.SearchField.index('Albums')) + if 'uuid' in cached_book: + hits = lib_books.Search(cached_book['uuid'],self.SearchField.index('Albums')) + if hits: + hit = hits[0] + if DEBUG: + self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Album)) + return hit + + hits = lib_books.Search(cached_book['author'],self.SearchField.index('Artists')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Album)) - return hit + if hit.Name == cached_book['title']: + if DEBUG: + self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Album)) + return hit attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) + self.log.warning(" attempt #%d" % (10 - attempts)) if DEBUG: self.log.error(" search for '%s' yielded no hits" % cached_book['title']) @@ -1544,10 +1568,11 @@ class ITUNES(DevicePlugin): exploded_file_size = 0 for file in myZipList: exploded_file_size += file.file_size - if DEBUG: + if False: self.log.info(" ITUNES._get_device_book_size()") self.log.info(" %d items in archive" % len(myZipList)) self.log.info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size)) + myZip.close() return exploded_file_size def _get_device_books(self): @@ -1578,9 +1603,11 @@ class ITUNES(DevicePlugin): self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) else: if DEBUG: - self.log.info(" adding %-30.30s %-30.30s [%s] %s" % - (book.name(), book.artist(), book.kind(), book.album())) + self.log.info(" %-30.30s %-30.30s %s [%s]" % + (book.name(), book.artist(), book.album(), book.kind())) device_books.append(book) + if DEBUG: + self.log.info() elif iswindows: if 'iPod' in self.sources: @@ -1608,8 +1635,10 @@ class ITUNES(DevicePlugin): self.log.info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) else: if DEBUG: - self.log.info(" adding %-30.30s %-30.30s [%s]" % (book.Name, book.Artist, book.KindAsString)) + self.log.info(" %-30.30s %-30.30s %s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) device_books.append(book) + if DEBUG: + self.log.info() finally: pythoncom.CoUninitialize() @@ -1642,6 +1671,7 @@ class ITUNES(DevicePlugin): If the database copy will be deleted after upload, we have to use file (the PersistentTemporaryFile), which will be around until calibre exits. + If we're using the database copy, delete the plist ''' if DEBUG: self.log.info(" ITUNES._get_fpath()") @@ -1649,18 +1679,25 @@ class ITUNES(DevicePlugin): fpath = file if not getattr(fpath, 'deleted_after_upload', False): if getattr(file, 'orig_file_path', None) is not None: + # Database copy fpath = file.orig_file_path + self._delete_iTunesMetadata_plist(fpath) elif getattr(file, 'name', None) is not None: + # PTF fpath = file.name else: + # Recipe - PTF if DEBUG: self.log.info(" file will be deleted after upload") + if update_md: - if DEBUG: - self.log.info(" metadata before rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) self._update_epub_metadata(fpath, metadata) - if DEBUG: - self.log.info(" metadata after rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) + +# if DEBUG: +# self.log.info(" metadata before rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) +# self._update_epub_metadata(fpath, metadata) +# if DEBUG: +# self.log.info(" metadata after rewrite: '{0[0]}' '{0[1]}' '{0[2]}'".format(self._dump_epub_metadata(fpath))) return fpath def _get_library_books(self): @@ -1710,12 +1747,12 @@ class ITUNES(DevicePlugin): if str(book.description()).startswith(self.description_prefix): if book.location() == appscript.k.missing_value: library_orphans[path] = book - if DEBUG: + if False: self.log.info(" found iTunes PTF '%s' in Library|Books" % book.name()) library_books[path] = book if DEBUG: - self.log.info(" adding %-30.30s %-30.30s [%s]" % (book.name(), book.artist(), book.kind())) + self.log.info(" %-30.30s %-30.30s %s [%s]" % (book.name(), book.artist(), book.album(), book.kind())) else: if DEBUG: self.log.info(' no Library playlists') @@ -1728,10 +1765,10 @@ class ITUNES(DevicePlugin): for source in self.iTunes.sources: if source.Kind == self.Sources.index('Library'): lib = source - self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) + self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) break else: - self.log.error(" Library source not found") + self.log.error(" Library source not found") if lib is not None: lib_books = None @@ -1740,22 +1777,22 @@ class ITUNES(DevicePlugin): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + self.log.info(" Books playlist: '%s'" % (pl.Name)) lib_books = pl.Tracks break else: if DEBUG: - self.log.error(" no Library|Books playlist found") + self.log.error(" no Library|Books playlist found") else: if DEBUG: - self.log.error(" no Library playlists found") + self.log.error(" no Library playlists found") try: for book in lib_books: # This may need additional entries for international iTunes users if book.KindAsString in ['MPEG audio file']: if DEBUG: - self.log.info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) + self.log.info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) else: path = self.path_template % (book.Name, book.Artist) @@ -1763,12 +1800,12 @@ class ITUNES(DevicePlugin): if book.Description.startswith(self.description_prefix): if not book.Location: library_orphans[path] = book - if DEBUG: + if False: self.log.info(" found iTunes PTF '%s' in Library|Books" % book.Name) library_books[path] = book if DEBUG: - self.log.info(" adding %-30.30s %-30.30s [%s]" % (book.Name, book.Artist, book.KindAsString)) + self.log.info(" %-30.30s %-30.30s %s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) except: if DEBUG: self.log.info(" no books in library") @@ -1902,7 +1939,7 @@ class ITUNES(DevicePlugin): This occurs when the user deletes a book in iBooks while disconnected ''' if DEBUG: - self.log.info("\n ITUNES._purge_orphans()") + self.log.info(" ITUNES._purge_orphans()") #self._dump_library_books(library_books) #self.log.info(" cached_books:\n %s" % "\n ".join(cached_books.keys())) @@ -1971,7 +2008,7 @@ class ITUNES(DevicePlugin): ''' self.log.info(" ITUNES._remove_from_device()") if isosx: - if DEBUG: + if False: self.log.info(" deleting %s" % cached_book['dev_book']) cached_book['dev_book'].delete() @@ -1980,7 +2017,7 @@ class ITUNES(DevicePlugin): hits = dev_pl.Search(cached_book['uuid'],self.SearchField.index('Albums')) if hits: hit = hits[0] - if DEBUG: + if False: self.log.info(" deleting '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Album)) hit.Delete() @@ -2020,7 +2057,7 @@ class ITUNES(DevicePlugin): self.log.info(" author_storage_path not empty (%d objects):" % len(author_files)) self.log.info(" %s" % '\n'.join(author_files)) else: - self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title']) + self.log.info(" '%s' (stored external to iTunes, no files deleted)" % cached_book['title']) except: # We get here if there was an error with .location().path @@ -2038,27 +2075,29 @@ class ITUNES(DevicePlugin): path = book.Location except: book = self._find_library_book(cached_book) - path = book.Location - storage_path = os.path.split(book.Location) - if book.Location.startswith(self.iTunes_media): - if DEBUG: - self.log.info(" removing '%s' at %s" % - (cached_book['title'], path)) - try: - os.remove(path) - except: - self.log.warning(" could not find '%s' in iTunes storage" % path) - try: - os.rmdir(storage_path[0]) - self.log.info(" removed folder '%s'" % storage_path[0]) - except: - self.log.info(" folder '%s' not found or not empty" % storage_path[0]) + if book.Location: + storage_path = os.path.split(book.Location) + if book.Location.startswith(self.iTunes_media): + if DEBUG: + self.log.info(" removing '%s' at %s" % + (cached_book['title'], book.Location)) + try: + os.remove(path) + except: + self.log.warning(" could not find '%s' in iTunes storage" % path) + try: + os.rmdir(storage_path[0]) + self.log.info(" removed folder '%s'" % storage_path[0]) + except: + self.log.info(" folder '%s' not found or not empty" % storage_path[0]) - # Delete from iTunes database + # Delete from iTunes database + else: + self.log.info(" '%s' (stored external to iTunes, no files deleted)" % cached_book['title']) else: - self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title']) - + if DEBUG: + self.log.info(" unable to find Library book '%s'" % cached_book['title']) book.Delete() def _update_epub_metadata(self, fpath, metadata): @@ -2068,16 +2107,43 @@ class ITUNES(DevicePlugin): # Refresh epub metadata with open(fpath,'r+b') as zfo: - + ''' # Touch the timestamp to force a recache if metadata.timestamp: + if DEBUG: + self.log.info(" old timestamp: %s" % metadata.timestamp) old_ts = metadata.timestamp metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) + if DEBUG: + self.log.info(" new timestamp: %s" % metadata.timestamp) else: metadata.timestamp = isoformat(now()) + if DEBUG: + self.log.info(" add timestamp: %s" % metadata.timestamp) + ''' + # Touch the OPF timestamp + zf_opf = ZipFile(fpath,'r') + fnames = zf_opf.namelist() + opf = [x for x in fnames if '.opf' in x][0] + if opf: + opf_raw = cStringIO.StringIO(zf_opf.read(opf)).getvalue() + soup = BeautifulSoup(opf_raw) + md = soup.find('metadata') + ts = md.find('meta',attrs={'name':'calibre:timestamp'}) + if ts: + # Touch existing calibre timestamp + timestamp = ts['content'] + old_ts = parse_date(timestamp) + metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, + old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) + else: + metadata.timestamp = isoformat(now()) + if DEBUG: + self.log.info(" add timestamp: %s" % metadata.timestamp) + zf_opf.close() - # Tweak the author if 'News' in tags for friendlier display in iBooks + # If 'News' in tags, tweak the title/author for friendlier display in iBooks if _('News') in metadata.tags: if metadata.title.find('[') > 0: metadata.title = metadata.title[:metadata.title.find('[')-1] @@ -2086,6 +2152,15 @@ class ITUNES(DevicePlugin): sort_author = re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', metadata.title).rstrip() metadata.author_sort = '%s %s' % (sort_author, strftime('%Y-%m-%d')) + # Remove any non-alpha category tags + for tag in metadata.tags: + if not self._is_alpha(tag[0]): + metadata.tags.remove(tag) + + # If windows & series, nuke tags so series used as Category during _update_iTunes_metadata() + if iswindows and metadata.series: + metadata.tags = None + set_metadata(zfo,metadata) def _update_device(self, msg='', wait=True): @@ -2146,19 +2221,15 @@ class ITUNES(DevicePlugin): if isosx: if lb_added: lb_added.album.set(metadata.uuid) - #lb_added.artist.set(metadata.authors[0]) lb_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.enabled.set(True) - lb_added.name.set(metadata.title) lb_added.sort_artist.set(metadata.author_sort.title()) lb_added.sort_name.set(this_book.title_sorter) if db_added: db_added.album.set(metadata.uuid) - #db_added.artist.set(metadata.authors[0]) db_added.description.set("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.enabled.set(True) - db_added.name.set(metadata.title) db_added.sort_artist.set(metadata.author_sort.title()) db_added.sort_name.set(this_book.title_sorter) @@ -2189,7 +2260,7 @@ class ITUNES(DevicePlugin): db_added.genre.set(metadata.series) db_added.episode_ID.set(metadata.series) db_added.episode_number.set(metadata.series_index) - else: + elif metadata.tags: for tag in metadata.tags: if self._is_alpha(tag[0]): if lb_added: @@ -2201,20 +2272,15 @@ class ITUNES(DevicePlugin): elif iswindows: if lb_added: lb_added.Album = metadata.uuid - #lb_added.Artist = metadata.authors[0] lb_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) lb_added.Enabled = True - lb_added.Name = metadata.title lb_added.SortArtist = (metadata.author_sort.title()) lb_added.SortName = (this_book.title_sorter) if db_added: - # Album, Artist and Name are changed in _add_device_book() db_added.Album = metadata.uuid - #db_added.Artist = metadata.authors[0] db_added.Description = ("%s %s" % (self.description_prefix,strftime('%Y-%m-%d %H:%M:%S'))) db_added.Enabled = True - db_added.Name = metadata.title db_added.SortArtist = (metadata.author_sort.title()) db_added.SortName = (this_book.title_sorter) @@ -2232,15 +2298,32 @@ class ITUNES(DevicePlugin): if db_added: db_added.AlbumRating = (metadata.rating*10) except: - pass + if DEBUG: + self.log.warning(" iTunes automation interface reported an error" + " setting AlbumRating") # Set Category from first alpha tag, overwrite with series if available # Otherwise iBooks uses first from opf # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12) - if metadata.tags: - if DEBUG: - self.log.info(" setting Category from metadata.tags") + if metadata.series: + if lb_added: + lb_added.Category = metadata.series + lb_added.EpisodeID = metadata.series + try: + lb_added.EpisodeNumber = metadata.series_index + except: + pass + if db_added: + db_added.Category = metadata.series + db_added.EpisodeID = metadata.series + try: + db_added.EpisodeNumber = metadata.series_index + except: + if DEBUG: + self.log.warning(" iTunes automation interface reported an error" + " setting EpisodeNumber") + elif metadata.tags: for tag in metadata.tags: if self._is_alpha(tag[0]): if lb_added: @@ -2249,27 +2332,6 @@ class ITUNES(DevicePlugin): db_added.Category = tag break - if metadata.series: - if DEBUG: - self.log.info(" setting Category from metadata.series") - if lb_added: - if DEBUG: - self.log.info(" setting lb_added from metadata.series") - lb_added.Category = metadata.series - lb_added.EpisodeID = metadata.series - try: - lb_added.EpisodeNumber = metadata.series_index - except: - pass - if db_added: - if DEBUG: - self.log.info(" setting db_added from metadata.series") - db_added.Category = metadata.series - db_added.EpisodeID = metadata.series - try: - db_added.EpisodeNumber = metadata.series_index - except: - pass class BookList(list): ''' From 11d8cfb6c97f35146a5895491736be81308606b3 Mon Sep 17 00:00:00 2001 From: GRiker Date: Thu, 17 Jun 2010 11:27:12 -0600 Subject: [PATCH 7/8] GwR revisions --- src/calibre/devices/apple/driver.py | 40 +++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 3c4f60edb7..1a7221e687 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -546,8 +546,24 @@ class ITUNES(DevicePlugin): else: self.log.info(" skipping sync phase, manual_sync_mode: True") else: - self.problem_titles.append("'%s' by %s" % - (self.cached_books[path]['title'],self.cached_books[path]['author'])) + if self.manual_sync_mode: + metadata = MetaInformation(self.cached_books[path]['title'], + [self.cached_books[path]['author']]) + metadata.uuid = self.cached_books[path]['uuid'] + + if isosx: + self._remove_existing_copy(self.cached_books[path],metadata) + elif iswindows: + try: + pythoncom.CoInitialize() + self.iTunes = win32com.client.Dispatch("iTunes.Application") + self._remove_existing_copy(self.cached_books[path],metadata) + finally: + pythoncom.CoUninitialize() + + else: + self.problem_titles.append("'%s' by %s" % + (self.cached_books[path]['title'],self.cached_books[path]['author'])) def eject(self): ''' @@ -800,7 +816,7 @@ class ITUNES(DevicePlugin): if isosx: for (i,file) in enumerate(files): path = self.path_template % (metadata[i].title, metadata[i].author[0]) - self._remove_existing_copies(path, metadata[i]) + self._remove_existing_copy(path, metadata[i]) fpath = self._get_fpath(file, metadata[i], update_md=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added) @@ -827,7 +843,7 @@ class ITUNES(DevicePlugin): for (i,file) in enumerate(files): path = self.path_template % (metadata[i].title, metadata[i].author[0]) - self._remove_existing_copies(path, metadata[i]) + self._remove_existing_copy(path, metadata[i]) fpath = self._get_fpath(file, metadata[i], update_md=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) @@ -1965,11 +1981,11 @@ class ITUNES(DevicePlugin): if DEBUG: self.log.info() - def _remove_existing_copies(self, path, metadata): + def _remove_existing_copy(self, path, metadata): ''' ''' if DEBUG: - self.log.info(" ITUNES._remove_existing_copies()") + self.log.info(" ITUNES._remove_existing_copy()") if self.manual_sync_mode: # Delete existing from Device|Books, add to self.update_list @@ -2063,7 +2079,11 @@ class ITUNES(DevicePlugin): # We get here if there was an error with .location().path self.log.info(" removing orphan '%s' from iTunes" % cached_book['title']) - self.iTunes.delete(cached_book['lib_book']) + try: + self.iTunes.delete(cached_book['lib_book']) + except: + if DEBUG: + self.log.info(" book not found in iTunes") elif iswindows: ''' @@ -2098,7 +2118,11 @@ class ITUNES(DevicePlugin): else: if DEBUG: self.log.info(" unable to find Library book '%s'" % cached_book['title']) - book.Delete() + try: + book.Delete() + except: + if DEBUG: + self.log.info(" book not found in iTunes") def _update_epub_metadata(self, fpath, metadata): ''' From 9d5598a61e3785dd319cec3216daae71c912e216 Mon Sep 17 00:00:00 2001 From: GRiker Date: Thu, 17 Jun 2010 11:40:10 -0600 Subject: [PATCH 8/8] GwR revisions --- src/calibre/devices/apple/driver.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 1a7221e687..831df4cdb8 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -2077,13 +2077,14 @@ class ITUNES(DevicePlugin): except: # We get here if there was an error with .location().path - self.log.info(" removing orphan '%s' from iTunes" % cached_book['title']) + if DEBUG: + self.log.info(" '%s' not found in iTunes" % cached_book['title']) try: self.iTunes.delete(cached_book['lib_book']) except: if DEBUG: - self.log.info(" book not found in iTunes") + self.log.info(" '%s' not found in iTunes" % cached_book['title']) elif iswindows: ''' @@ -2096,7 +2097,7 @@ class ITUNES(DevicePlugin): except: book = self._find_library_book(cached_book) - if book.Location: + if book: storage_path = os.path.split(book.Location) if book.Location.startswith(self.iTunes_media): if DEBUG: @@ -2117,12 +2118,12 @@ class ITUNES(DevicePlugin): self.log.info(" '%s' (stored external to iTunes, no files deleted)" % cached_book['title']) else: if DEBUG: - self.log.info(" unable to find Library book '%s'" % cached_book['title']) + self.log.info(" '%s' not found in iTunes" % cached_book['title']) try: book.Delete() except: if DEBUG: - self.log.info(" book not found in iTunes") + self.log.info(" '%s' not found in iTunes" % cached_book['title']) def _update_epub_metadata(self, fpath, metadata): '''