Merge from trunk

This commit is contained in:
Charles Haley 2010-06-07 19:08:55 +01:00
commit 93a751cc78
5 changed files with 191 additions and 154 deletions

View File

@ -5,12 +5,13 @@ __copyright__ = '2010, Gregory Riker'
__docformat__ = 'restructuredtext en'
import cStringIO, os, re, shutil, sys, tempfile, time, zipfile
import cStringIO, os, re, shutil, subprocess, sys, tempfile, time, zipfile
from calibre.constants import DEBUG
from calibre import fit_image
from calibre.constants import isosx, iswindows
from calibre.devices.interface import DevicePlugin
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.ebooks.metadata import MetaInformation
from calibre.library.server.utils import strftime
from calibre.ptempfile import PersistentTemporaryFile
@ -106,6 +107,7 @@ class ITUNES(DevicePlugin):
cache_dir = os.path.join(config_dir, 'caches', 'itunes')
ejected = False
iTunes= None
iTunes_media = None
log = Log()
path_template = 'iTunes/%s - %s.epub'
problem_titles = []
@ -138,12 +140,12 @@ class ITUNES(DevicePlugin):
self.log.info( "ITUNES.add_books_to_metadata()")
self._dump_update_list('add_books_to_metadata()')
for (j,p_book) in enumerate(self.update_list):
self.log.info("ITUNES.add_books_to_metadata(): looking for %s" %
self.log.info("ITUNES.add_books_to_metadata():\n looking for %s" %
str(p_book['lib_book'])[-9:])
for i,bl_book in enumerate(booklists[0]):
if bl_book.library_id == p_book['lib_book']:
booklists[0].pop(i)
self.log.info("ITUNES.add_books_to_metadata(): removing %s %s" %
self.log.info("ITUNES.add_books_to_metadata():\n removing %s %s" %
(p_book['title'], str(p_book['lib_book'])[-9:]))
break
else:
@ -180,8 +182,9 @@ class ITUNES(DevicePlugin):
self.log.info(" adding '%s' by '%s' to booklists[0]" %
(new_book.title, new_book.author))
booklists[0].append(new_book)
if DEBUG:
self._dump_booklist(booklists[0],'after add_books_to_metadata()')
# if DEBUG:
# self._dump_booklist(booklists[0],'after add_books_to_metadata()')
def books(self, oncard=None, end_session=True):
"""
@ -595,7 +598,7 @@ class ITUNES(DevicePlugin):
L{books}(oncard='cardb')).
'''
if DEBUG:
self.log.info("ITUNES.remove_books_from_metadata():")
self.log.info("ITUNES.remove_books_from_metadata()")
for path in paths:
if self.cached_books[path]['lib_book']:
# Remove from the booklist
@ -605,15 +608,15 @@ class ITUNES(DevicePlugin):
booklists[0].pop(i)
break
else:
self.log.error("ITUNES.remove_books_from_metadata(): '%s' not found in self.cached_book" % path)
self.log.error(" '%s' not found in self.cached_book" % path)
# Remove from cached_books
self.cached_books.pop(path)
if DEBUG:
self.log.info("ITUNES.remove_books_from_metadata(): Removing '%s' from self.cached_books" % path)
self._dump_cached_books('remove_books_from_metadata()')
self.log.info(" Removing '%s' from self.cached_books" % path)
# self._dump_cached_books('remove_books_from_metadata()')
else:
self.log.warning("ITUNES.remove_books_from_metadata(): skipping purchased book, can't remove via automation interface")
self.log.warning(" skipping purchased book, can't remove via automation interface")
def reset(self, key='-1', log_packets=False, report_progress=None,
detected_device=None) :
@ -657,54 +660,13 @@ class ITUNES(DevicePlugin):
L{books}(oncard='cardb')).
'''
if DEBUG:
self.log.info("ITUNES:sync_booklists():")
self.log.info("ITUNES:sync_booklists()")
if self.update_needed:
if DEBUG:
self.log.info(' calling _update_device')
self._update_device(msg=self.update_msg, wait=False)
self.update_needed = False
# Get actual size of updated books on device
if self.update_list:
if DEBUG:
self._dump_update_list(header='sync_booklists()')
if isosx:
for updated_book in self.update_list:
size_on_device = self._get_device_book_size(updated_book['title'],
updated_book['author'][0])
if size_on_device:
for book in booklists[0]:
if book.title == updated_book['title'] and \
book.author == updated_book['author']:
break
else:
self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
else:
self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
elif iswindows:
try:
pythoncom.CoInitialize()
self.iTunes = win32com.client.Dispatch("iTunes.Application")
for updated_book in self.update_list:
size_on_device = self._get_device_book_size(updated_book['title'], updated_book['author'])
if size_on_device:
for book in booklists[0]:
if book.title == updated_book['title'] and \
book.author[0] == updated_book['author']:
book.size = size_on_device
break
else:
self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
else:
self.log.error("ITUNES:sync_booklists(): could not find '%s' on device" % updated_book['title'])
finally:
pythoncom.CoUninitialize()
self.update_list = []
self.update_needed = False
# Inform user of any problem books
if self.problem_titles:
@ -763,22 +725,49 @@ class ITUNES(DevicePlugin):
"Click 'Show Details' for a list.")
if DEBUG:
self.log.info("ITUNES.upload_books():")
self.log.info("ITUNES.upload_books()")
self._dump_files(files, header='upload_books()')
self._dump_cached_books('upload_books()')
# self._dump_cached_books('upload_books()')
self._dump_update_list('upload_books()')
if isosx:
for (i,file) in enumerate(files):
# Delete existing from Library|Books
# Add to self.update_list for deletion from booklist[0] during add_books_to_metadata
'''
# ---------------------------
# PROVISIONAL
# Use the cover to find the database storage point of the epub
# Pass database copy to iTunes instead of the temporary file
if False:
if DEBUG:
self.log.info(" processing '%s'" % metadata[i].title)
self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
self.log.info(" cover: %s" % metadata[i].cover)
calibre_database_item = False
if metadata[i].cover:
passed_file = file
storage_path = os.path.split(metadata[i].cover)[0]
try:
database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
file = os.path.join(storage_path,database_epub)
calibre_database_item = True
self.log.info(" using database file: %s" % file)
except:
self.log.info(" could not find epub in %s" % storage_path)
else:
self.log.info(" no cover available, using temp file")
# ---------------------------
'''
path = self.path_template % (metadata[i].title, metadata[i].author[0])
# Delete existing from Library|Books, add to self.update_list
# for deletion from booklist[0] during add_books_to_metadata
if path in self.cached_books:
if DEBUG:
self.log.info(" adding '%s' by %s to self.update_list" %
(self.cached_books[path]['title'],self.cached_books[path]['author']))
# *** Second time a book is updated the author is a list ***
self.update_list.append(self.cached_books[path])
if DEBUG:
@ -826,16 +815,17 @@ class ITUNES(DevicePlugin):
this_book.device_collections = []
this_book.library_id = added
this_book.path = path
this_book.size = added.size() # Updated later from actual storage size
this_book.size = self._get_device_book_size(file, added.size())
this_book.thumbnail = thumb
this_book.iTunes_id = added
new_booklist.append(this_book)
# Flesh out the iTunes metadata
added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
# Populate the iTunes metadata
if metadata[i].comments:
added.comment.set(strip_tags.sub('',metadata[i].comments))
added.description.set("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
added.enabled.set(True)
if metadata[i].rating:
added.rating.set(metadata[i].rating*10)
added.sort_artist.set(metadata[i].author_sort.title())
@ -859,6 +849,7 @@ class ITUNES(DevicePlugin):
# Report progress
if self.report_progress is not None:
self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count))
elif iswindows:
try:
pythoncom.CoInitialize()
@ -886,18 +877,39 @@ class ITUNES(DevicePlugin):
if DEBUG:
self.log.error(" no Books playlist found")
#
# lib = self.iTunes.sources.ItemByName('Library')
# lib_playlists = [pl.Name for pl in lib.Playlists]
# if not 'Books' in lib_playlists:
# self.log.error(" no 'Books' playlist in Library")
# library_books = lib.Playlists.ItemByName('Books')
#
for (i,file) in enumerate(files):
path = self.path_template % (metadata[i].title, metadata[i].author[0])
# Delete existing from Library|Books, add to self.update_list
# for deletion from booklist[0] during add_books_to_metadata
'''
# ---------------------------
# PROVISIONAL
# Use the cover to find the database storage point of the epub
# Pass database copy to iTunes instead of the temporary file
if False:
if DEBUG:
self.log.info(" processing '%s'" % metadata[i].title)
self.log.info(" file: %s" % (file._name if isinstance(file,PersistentTemporaryFile) else file))
self.log.info(" cover: %s" % metadata[i].cover)
calibre_database_item = False
if metadata[i].cover:
passed_file = file
storage_path = os.path.split(metadata[i].cover)[0]
try:
database_epub = filter(lambda x: x.endswith('.epub'), os.listdir(storage_path))[0]
file = os.path.join(storage_path,database_epub)
calibre_database_item = True
self.log.info(" using database file: %s" % file)
except:
self.log.info(" could not find epub in %s" % storage_path)
else:
self.log.info(" no cover available, using temp file")
# ---------------------------
'''
path = self.path_template % (metadata[i].title, metadata[i].author[0])
if path in self.cached_books:
self.update_list.append(self.cached_books[path])
@ -995,9 +1007,10 @@ class ITUNES(DevicePlugin):
new_booklist.append(this_book)
# Flesh out the iTunes metadata
added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
if metadata[i].comments:
added.Comment = (strip_tags.sub('',metadata[i].comments))
added.Description = ("added by calibre %s" % strftime('%Y-%m-%d %H:%M:%S'))
added.Enabled = True
if metadata[i].rating:
added.AlbumRating = (metadata[i].rating*10)
added.SortArtist = (metadata[i].author_sort.title())
@ -1150,7 +1163,7 @@ class ITUNES(DevicePlugin):
hits = lib_books.Search(cached_book['author'],SearchField.index('Artists'))
if hits:
for hit in hits:
self.log.info(" evaluating '%s' by %s" % (hit.Name, hit.Artist))
#self.log.info(" evaluating '%s' by %s" % (hit.Name, hit.Artist))
if hit.Name == cached_book['title']:
self.log.info(" matched '%s' by %s" % (hit.Name, hit.Artist))
return hit
@ -1234,34 +1247,20 @@ class ITUNES(DevicePlugin):
self.log.error(" error generating thumb for '%s'" % book.Name)
return None
def _get_device_book_size(self, title, author):
def _get_device_book_size(self, file, compressed_size):
'''
Fetch the size of a book stored on the device
Calculate the exploded size of file
'''
myZip = zipfile.ZipFile(file,'r')
myZipList = myZip.infolist()
exploded_file_size = 0
for file in myZipList:
exploded_file_size += file.file_size
if DEBUG:
self.log.info("ITUNES._get_device_book_size():\n looking for title: '%s' author: '%s'" %
(title,author))
device_books = self._get_device_books()
if isosx:
for d_book in device_books:
if d_book.name() == title and d_book.artist() == author:
if DEBUG:
self.log.info(' found it')
return d_book.size()
else:
self.log.error("ITUNES._get_device_book_size():"
" could not find '%s' by '%s' in device_books" % (title,author))
return None
elif iswindows:
for d_book in device_books:
if d_book.Name == title and d_book.Artist == author:
self.log.info(" found it")
return d_book.Size
else:
self.log.error(" could not find '%s' by '%s' in device_books" % (title,author))
return None
self.log.info("ITUNES._get_device_book_size()")
self.log.info(" %d items in archive" % len(myZipList))
self.log.info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size))
return exploded_file_size
def _get_device_books(self):
'''
@ -1360,7 +1359,6 @@ class ITUNES(DevicePlugin):
if DEBUG:
self.log.info('ITUNES._get_library_books():\n No Books playlist')
elif iswindows:
lib = None
try:
@ -1463,11 +1461,21 @@ class ITUNES(DevicePlugin):
self.iTunes = appscript.app('iTunes')
initial_status = 'already running'
# Read the current storage path for iTunes media
cmd = "defaults read com.apple.itunes NSNavLastRootDirectory"
proc = subprocess.Popen( cmd, shell=True, cwd=os.curdir, stdout=subprocess.PIPE)
retcode = proc.wait()
media_dir = os.path.abspath(proc.communicate()[0].strip())
if os.path.exists(media_dir):
self.iTunes_media = media_dir
else:
self.log.error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes')
if DEBUG:
self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
self.log.info(" [%s - %s (%s), driver version %d.%d.%d]" %
(self.iTunes.name(), self.iTunes.version(), initial_status,
self.version[0],self.version[1],self.version[2]))
self.log.info(" iTunes_media: %s" % self.iTunes_media)
if iswindows:
'''
Launch iTunes if not already running
@ -1479,40 +1487,59 @@ class ITUNES(DevicePlugin):
self.iTunes.Windows[0].Minimized = True
initial_status = 'launched'
# Read the current storage path for iTunes media from the XML file
with open(self.iTunes.LibraryXMLPath, 'r') as xml:
soup = BeautifulSoup(xml.read().decode('utf-8'))
mf = soup.find('key',text="Music Folder").parent
string = mf.findNext('string').renderContents()
media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
if os.path.exists(media_dir):
self.iTunes_media = media_dir
else:
self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath)
self.log.error(" %s" % string.parent.prettify())
self.log.error(" '%s' not found" % media_dir)
if DEBUG:
self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
(self.iTunes.Windows[0].name, self.iTunes.Version, initial_status,
self.version[0],self.version[1],self.version[2]))
self.log.info(" iTunes_media: %s" % self.iTunes_media)
def _remove_from_iTunes(self, cached_book):
'''
iTunes does not delete books from storage when removing from database
We only want to delete stored copies if the file is stored in iTunes
We don't want to delete files stored outside of iTunes
'''
if isosx:
storage_path = os.path.split(cached_book['lib_book'].location().path)
title_storage_path = storage_path[0]
if DEBUG:
self.log.info("ITUNES._remove_from_iTunes():")
self.log.info(" removing title_storage_path: %s" % title_storage_path)
try:
shutil.rmtree(title_storage_path)
except:
self.log.info(" '%s' not empty" % title_storage_path)
if cached_book['lib_book'].location().path.startswith(self.iTunes_media):
title_storage_path = storage_path[0]
if DEBUG:
self.log.info("ITUNES._remove_from_iTunes():")
self.log.info(" removing title_storage_path: %s" % title_storage_path)
try:
shutil.rmtree(title_storage_path)
except:
self.log.info(" '%s' not empty" % title_storage_path)
# Clean up title/author directories
author_storage_path = os.path.split(title_storage_path)[0]
self.log.info(" author_storage_path: %s" % author_storage_path)
author_files = os.listdir(author_storage_path)
if '.DS_Store' in author_files:
author_files.pop(author_files.index('.DS_Store'))
if not author_files:
shutil.rmtree(author_storage_path)
if DEBUG:
self.log.info(" removing empty author_storage_path")
# Clean up title/author directories
author_storage_path = os.path.split(title_storage_path)[0]
self.log.info(" author_storage_path: %s" % author_storage_path)
author_files = os.listdir(author_storage_path)
if '.DS_Store' in author_files:
author_files.pop(author_files.index('.DS_Store'))
if not author_files:
shutil.rmtree(author_storage_path)
if DEBUG:
self.log.info(" removing empty author_storage_path")
else:
if DEBUG:
self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
self.log.info(" %s" % '\n'.join(author_files))
else:
if DEBUG:
self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
self.log.info(" %s" % '\n'.join(author_files))
self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title'])
self.iTunes.delete(cached_book['lib_book'])
@ -1522,26 +1549,34 @@ class ITUNES(DevicePlugin):
Windows stores the book under a common author directory, so we just delete the .epub
'''
if DEBUG:
self.log.info("ITUNES._remove_from_iTunes(): '%s'" % cached_book['title'])
self.log.info("ITUNES._remove_from_iTunes():\n '%s'" % cached_book['title'])
book = self._find_library_book(cached_book)
if book:
if DEBUG:
self.log.info("ITUNES._remove_from_iTunes():\n deleting '%s' at %s" %
(cached_book['title'], book.Location))
folder = os.path.split(book.Location)[0]
path = book.Location
storage_path = os.path.split(book.Location)
if book.Location.startswith(self.iTunes_media):
if DEBUG:
self.log.info("ITUNES._remove_from_iTunes():")
self.log.info(" removing '%s' at %s" %
(cached_book['title'], path))
try:
os.remove(path)
except:
self.log.warning(" could not find '%s' in iTunes storage" % path)
try:
os.rmdir(storage_path[0])
self.log.info(" removed folder '%s'" % storage_path[0])
except:
self.log.info(" folder '%s' not found or not empty" % storage_path[0])
# Delete from iTunes database
else:
self.log.info(" '%s' stored external to iTunes, no files deleted" % cached_book['title'])
book.Delete()
try:
os.remove(path)
except:
self.log.warning(" could not find '%s' in iTunes storage" % path)
try:
os.rmdir(folder)
self.log.info(" removed folder '%s'" % folder)
except:
self.log.info(" folder '%s' not found or not empty" % folder)
else:
self.log.warning(" could not find '%s' in iTunes storage" % cached_book['title'])
self.log.warning(" could not find '%s' in iTunes database" % cached_book['title'])
def _update_device(self, msg='', wait=True):
'''

View File

@ -108,8 +108,6 @@ class XMLCache(object):
seen = set([])
for item in list(pl):
id_ = item.get('id', None)
# if id_ is None or id_ in seen or not root.xpath(
# '//*[local-name()!="item" and @id="%s"]'%id_):
if id_ is None or id_ in seen or id_map.get(id_, None) is None:
if DEBUG:
if id_ is None:
@ -167,11 +165,6 @@ class XMLCache(object):
items = []
for item in playlist:
id_ = item.get('id', None)
# records = root.xpath(
# '//*[local-name()="text" and @id="%s"]'%id_)
# if records:
# if records is not None:
# items.append(records[0])
record = id_map.get(id_, None)
if record is not None:
items.append(record)
@ -297,7 +290,6 @@ class XMLCache(object):
lpath_map = self.build_lpath_map(root)
for book in bl:
#record = self.book_by_lpath(book.lpath, root)
record = lpath_map[book.lpath]
if record is not None:
title = record.get('title', None)
@ -367,7 +359,6 @@ class XMLCache(object):
collections = booklist.get_collections(collections_attributes)
lpath_map = self.build_lpath_map(root)
for category, books in collections.items():
# records = [self.book_by_lpath(b.lpath, root) for b in books]
records = [lpath_map.get(b.lpath, None) for b in books]
# Remove any books that were not found, although this
# *should* never happen
@ -409,7 +400,6 @@ class XMLCache(object):
playlist.getparent().remove(playlist)
continue
books = collections[title]
# records = [self.book_by_lpath(b.lpath, root) for b in books]
records = [lpath_map.get(b.lpath, None) for b in books]
records = [x for x in records if x is not None]
ids = [x.get('id', None) for x in records]

View File

@ -122,7 +122,7 @@ class DeviceManager(Thread):
try:
dev.open()
except:
print 'Unable to open device', dev
prints('Unable to open device', str(dev))
traceback.print_exc()
continue
self.connected_device = dev
@ -168,11 +168,11 @@ class DeviceManager(Thread):
if possibly_connected_devices:
if not self.do_connect(possibly_connected_devices,
is_folder_device=False):
print 'Connect to device failed, retrying in 5 seconds...'
prints('Connect to device failed, retrying in 5 seconds...')
time.sleep(5)
if not self.do_connect(possibly_connected_devices,
is_folder_device=False):
print 'Device connect failed again, giving up'
prints('Device connect failed again, giving up')
def umount_device(self, *args):
if self.is_device_connected and not self.job_manager.has_device_jobs():
@ -317,10 +317,8 @@ class DeviceManager(Thread):
def _save_books(self, paths, target):
'''Copy books from device to disk'''
for path in paths:
# name = path.rpartition(getattr(self.device, 'path_sep', '/'))[2]
name = path.rpartition(os.sep)[2]
dest = os.path.join(target, name)
print path, dest
if os.path.abspath(dest) != os.path.abspath(path):
f = open(dest, 'wb')
self.device.get_file(path, f)

View File

@ -171,8 +171,7 @@ def add_borders_to_image(path_to_image, left=0, top=0, right=0, bottom=0,
border_color)
compose_image(canvas, img, left, top)
p.DestroyMagickWand(img)
with open(path_to_image, 'wb') as f:
p.MagickWriteImage(canvas, f)
p.MagickWriteImage(canvas,path_to_image)
p.DestroyMagickWand(canvas)
def create_cover_page(top_lines, logo_path, width=590, height=750,

View File

@ -24,6 +24,7 @@ from calibre.ebooks.metadata import MetaInformation
from calibre.web.feeds import feed_from_xml, templates, feeds_from_index, Feed
from calibre.web.fetch.simple import option_parser as web2disk_option_parser
from calibre.web.fetch.simple import RecursiveFetcher
from calibre.utils.magick_draw import add_borders_to_image
from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.date import now as nowf
@ -283,6 +284,15 @@ class BasicNewsRecipe(Recipe):
#: Override this in your recipe to provide a url to use as a masthead.
masthead_url = None
#: By default, the cover image returned by get_cover_url() will be used as
#: the cover for the periodical. Overriding this in your recipe instructs
#: calibre to render the downloaded cover into a frame whose width and height
#: are expressed as a percentage of the downloaded cover.
#: cover_margins = (10,15,'white') pads the cover with a white margin
#: 10px on the left and right, 15px on the top and bottom.
#: Colors name defined at http://www.imagemagick.org/script/color.php
cover_margins = (0,0,'white')
#: Set to a non empty string to disable this recipe
#: The string will be used as the disabled message
recipe_disabled = None
@ -974,6 +984,11 @@ class BasicNewsRecipe(Recipe):
self.report_progress(1, _('Downloading cover from %s')%cu)
with nested(open(cpath, 'wb'), closing(self.browser.open(cu))) as (cfile, r):
cfile.write(r.read())
if self.cover_margins[0] or self.cover_margins[1]:
add_borders_to_image(cpath,
left=self.cover_margins[0],right=self.cover_margins[0],
top=self.cover_margins[1],bottom=self.cover_margins[1],
border_color=self.cover_margins[2])
if ext.lower() == 'pdf':
from calibre.ebooks.metadata.pdf import get_metadata
stream = open(cpath, 'rb')