Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Revisions to catalog building code, cleaning up diagnostics, tweaks to TOC section titles. Revisions to Apple driver in anticipation of releasing optional iDevice driver.

Browse Source
This commit is contained in:
GRiker 2012-12-03 16:49:09 -07:00
parent 3e5aed49f5
commit 8292b1d71d
3 changed files with 681 additions and 619 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
src/calibre/devices/apple/driver.py
View File

@ -20,6 +20,7 @@ from calibre.utils.config import config_dir, dynamic, prefs
from calibre.utils.date import now, parse_date from calibre.utils.date import now, parse_date
from calibre.utils.zipfile import ZipFile from calibre.utils.zipfile import ZipFile
def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None):
if not hasattr(dt, 'timetuple'): if not hasattr(dt, 'timetuple'):
@ -38,6 +39,7 @@ def logger():
_log = ThreadSafeLog() _log = ThreadSafeLog()
return _log return _log
class AppleOpenFeedback(OpenFeedback): class AppleOpenFeedback(OpenFeedback):
def __init__(self, plugin): def __init__(self, plugin):
@ -102,6 +104,7 @@ class AppleOpenFeedback(OpenFeedback):
return Dialog(parent, self) return Dialog(parent, self)
class DriverBase(DeviceConfig, DevicePlugin): class DriverBase(DeviceConfig, DevicePlugin):
# Needed for config_widget to work # Needed for config_widget to work
FORMATS = ['epub', 'pdf'] FORMATS = ['epub', 'pdf']
@ -133,11 +136,11 @@ class DriverBase(DeviceConfig, DevicePlugin):
False, False,
] ]
@classmethod @classmethod
def _config_base_name(cls): def _config_base_name(cls):
return 'iTunes' return 'iTunes'
class ITUNES(DriverBase): class ITUNES(DriverBase):
''' '''
Calling sequences: Calling sequences:
@ -158,6 +161,7 @@ class ITUNES(DriverBase):
can_handle() can_handle()
set_progress_reporter() set_progress_reporter()
books() (once for each storage point) books() (once for each storage point)
(create self.cached_books)
settings() settings()
settings() settings()
can_handle() (~1x per second OSX while idle) can_handle() (~1x per second OSX while idle)
@ -195,7 +199,7 @@ class ITUNES(DriverBase):
supported_platforms = ['osx', 'windows'] supported_platforms = ['osx', 'windows']
author = 'GRiker' author = 'GRiker'
#: The version of this plugin as a 3-tuple (major, minor, revision) #: The version of this plugin as a 3-tuple (major, minor, revision)
version = (1,0,0) version = (1, 1, 1)
DISPLAY_DISABLE_DIALOG = "display_disable_apple_driver_dialog" DISPLAY_DISABLE_DIALOG = "display_disable_apple_driver_dialog"
@ -409,7 +413,7 @@ class ITUNES(DriverBase):
else: else:
logger().info(" Cover fetching/caching disabled") logger().info(" Cover fetching/caching disabled")
# Fetch a list of books from iPod device connected to iTunes # Fetch a list of books from iDevice connected to iTunes
if 'iPod' in self.sources: if 'iPod' in self.sources:
booklist = BookList(logger()) booklist = BookList(logger())
cached_books = {} cached_books = {}
@ -440,7 +444,8 @@ class ITUNES(DriverBase):
cached_books[this_book.path] = { cached_books[this_book.path] = {
'title': book.name(), 'title': book.name(),
'author':book.artist().split(' & '), 'author': book.artist(),
'authors': book.artist().split(' & '),
'lib_book': library_books[this_book.path] if this_book.path in library_books else None, 'lib_book': library_books[this_book.path] if this_book.path in library_books else None,
'dev_book': book, 'dev_book': book,
'uuid': book.composer() 'uuid': book.composer()
@ -480,7 +485,8 @@ class ITUNES(DriverBase):
cached_books[this_book.path] = { cached_books[this_book.path] = {
'title': book.Name, 'title': book.Name,
'author':book.Artist.split(' & '), 'author': book.Artist,
'authors': book.Artist.split(' & '),
'lib_book': library_books[this_book.path] if this_book.path in library_books else None, 'lib_book': library_books[this_book.path] if this_book.path in library_books else None,
'uuid': book.Composer, 'uuid': book.Composer,
'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub' 'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
@ -556,7 +562,7 @@ class ITUNES(DriverBase):
self.sources = self._get_sources() self.sources = self._get_sources()
if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''): if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
attempts -= 1 attempts -= 1
time.sleep(0.5) time.sleep(1.0)
if DEBUG: if DEBUG:
logger().warning(" waiting for connected iDevice, attempt #%d" % (10 - attempts)) logger().warning(" waiting for connected iDevice, attempt #%d" % (10 - attempts))
else: else:
@ -634,9 +640,9 @@ class ITUNES(DriverBase):
self.sources = self._get_sources() self.sources = self._get_sources()
if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''): if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
attempts -= 1 attempts -= 1
time.sleep(0.5) time.sleep(1.0)
if DEBUG: if DEBUG:
logger().warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) logger().warning(" waiting for connected iDevice, attempt #%d" % (10 - attempts))
else: else:
if DEBUG: if DEBUG:
logger().info(' found connected iPad in iTunes') logger().info(' found connected iPad in iTunes')
@ -720,8 +726,11 @@ class ITUNES(DriverBase):
else: else:
if self.manual_sync_mode: if self.manual_sync_mode:
metadata = MetaInformation(self.cached_books[path]['title'], metadata = MetaInformation(self.cached_books[path]['title'],
[self.cached_books[path]['author']]) self.cached_books[path]['authors'])
metadata.author = self.cached_books[path]['author']
metadata.uuid = self.cached_books[path]['uuid'] metadata.uuid = self.cached_books[path]['uuid']
if not metadata.uuid:
metadata.uuid = "unknown"
if isosx: if isosx:
self._remove_existing_copy(self.cached_books[path], metadata) self._remove_existing_copy(self.cached_books[path], metadata)
@ -848,18 +857,18 @@ class ITUNES(DriverBase):
raise OpenFeedback(self.ITUNES_SANDBOX_LOCKOUT_MESSAGE) raise OpenFeedback(self.ITUNES_SANDBOX_LOCKOUT_MESSAGE)
if DEBUG: if DEBUG:
VENDOR_ID = "0x%x" % connected_device[0] vendor_id = "0x%x" % connected_device[0]
PRODUCT_ID = "0x%x" % connected_device[1] product_id = "0x%x" % connected_device[1]
BCD = "0x%x" % connected_device[2] bcd = "0x%x" % connected_device[2]
MFG = connected_device[3] mfg = connected_device[3]
MODEL = connected_device[4] model = connected_device[4]
logger().info("%s.open(MFG: %s, VENDOR_ID: %s, MODEL: %s, BCD: %s, PRODUCT_ID: %s)" % logger().info("%s.open(MFG: %s, VENDOR_ID: %s, MODEL: %s, BCD: %s, PRODUCT_ID: %s)" %
(self.__class__.__name__, (self.__class__.__name__,
MFG, mfg,
VENDOR_ID, vendor_id,
MODEL, model,
BCD, bcd,
PRODUCT_ID product_id
)) ))
# Display a dialog recommending using 'Connect to iTunes' if user hasn't # Display a dialog recommending using 'Connect to iTunes' if user hasn't
@ -870,6 +879,10 @@ class ITUNES(DriverBase):
if DEBUG: if DEBUG:
logger().error(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) logger().error(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE)
# Log supported DEVICE_IDs and BCDs
logger().info(" BCD: %s" % ['0x%x' % x for x in sorted(self.BCD)])
logger().info(" PRODUCT_ID: %s" % ['0x%x' % x for x in sorted(self.PRODUCT_ID)])
# Confirm/create thumbs archive # Confirm/create thumbs archive
if not os.path.exists(self.cache_dir): if not os.path.exists(self.cache_dir):
if DEBUG: if DEBUG:
@ -916,7 +929,7 @@ class ITUNES(DriverBase):
logger().info(" looking for '%s' by '%s' uuid:%s" % logger().info(" looking for '%s' by '%s' uuid:%s" %
(self.cached_books[path]['title'], (self.cached_books[path]['title'],
self.cached_books[path]['author'], self.cached_books[path]['author'],
self.cached_books[path]['uuid'])) repr(self.cached_books[path]['uuid'])))
# Purge the booklist, self.cached_books, thumb cache # Purge the booklist, self.cached_books, thumb cache
for i, bl_book in enumerate(booklists[0]): for i, bl_book in enumerate(booklists[0]):
@ -925,24 +938,28 @@ class ITUNES(DriverBase):
(bl_book.title, bl_book.author, bl_book.uuid)) (bl_book.title, bl_book.author, bl_book.uuid))
found = False found = False
if bl_book.uuid == self.cached_books[path]['uuid']: if bl_book.uuid and bl_book.uuid == self.cached_books[path]['uuid']:
if False: if True:
logger().info(" matched with uuid") logger().info(" --matched uuid")
booklists[0].pop(i) booklists[0].pop(i)
found = True found = True
elif bl_book.title == self.cached_books[path]['title'] and \ elif bl_book.title == self.cached_books[path]['title'] and \
bl_book.author[0] == self.cached_books[path]['author']: bl_book.author == self.cached_books[path]['author']:
if False: if True:
logger().info(" matched with title + author") logger().info(" --matched title + author")
booklists[0].pop(i) booklists[0].pop(i)
found = True found = True
if found: if found:
# Remove from self.cached_books # Remove from self.cached_books
for cb in self.cached_books: for cb in self.cached_books:
if self.cached_books[cb]['uuid'] == self.cached_books[path]['uuid']: if (self.cached_books[cb]['uuid'] == self.cached_books[path]['uuid'] and
self.cached_books[cb]['author'] == self.cached_books[path]['author'] and
self.cached_books[cb]['title'] == self.cached_books[path]['title']):
self.cached_books.pop(cb) self.cached_books.pop(cb)
break break
else:
logger().error(" '%s' not found in self.cached_books" % self.cached_books[path]['title'])
# Remove from thumb from thumb cache # Remove from thumb from thumb cache
thumb_path = path.rpartition('.')[0] + '.jpg' thumb_path = path.rpartition('.')[0] + '.jpg'
@ -965,7 +982,9 @@ class ITUNES(DriverBase):
else: else:
if DEBUG: if DEBUG:
logger().error(" unable to find '%s' by '%s' (%s)" % logger().error(" unable to find '%s' by '%s' (%s)" %
(bl_book.title, bl_book.author,bl_book.uuid)) (self.cached_books[path]['title'],
self.cached_books[path]['author'],
self.cached_books[path]['uuid']))
if False: if False:
self._dump_booklist(booklists[0], indent=2) self._dump_booklist(booklists[0], indent=2)
@ -1656,8 +1675,8 @@ class ITUNES(DriverBase):
for book in booklist: for book in booklist:
if isosx: if isosx:
logger().info("%s%-40.40s %-30.30s %-10.10s %s" % logger().info("%s%-40.40s %-30.30s %-40.40s %-10.10s" %
(' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) (' ' * indent, book.title, book.author, book.uuid, str(book.library_id)[-9:]))
elif iswindows: elif iswindows:
logger().info("%s%-40.40s %-30.30s" % logger().info("%s%-40.40s %-30.30s" %
(' ' * indent, book.title, book.author)) (' ' * indent, book.title, book.author))
@ -1706,13 +1725,14 @@ class ITUNES(DriverBase):
logger().info("%s%s" % (' ' * indent, '-' * len(msg))) logger().info("%s%s" % (' ' * indent, '-' * len(msg)))
if isosx: if isosx:
for cb in self.cached_books.keys(): for cb in self.cached_books.keys():
logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % logger().info("%s%-40.40s %-30.30s %-40.40s %-10.10s %-10.10s" %
(' ' * indent, (' ' * indent,
self.cached_books[cb]['title'], self.cached_books[cb]['title'],
self.cached_books[cb]['author'], self.cached_books[cb]['author'],
self.cached_books[cb]['uuid'],
str(self.cached_books[cb]['lib_book'])[-9:], str(self.cached_books[cb]['lib_book'])[-9:],
str(self.cached_books[cb]['dev_book'])[-9:], str(self.cached_books[cb]['dev_book'])[-9:],
self.cached_books[cb]['uuid'])) ))
elif iswindows: elif iswindows:
for cb in self.cached_books.keys(): for cb in self.cached_books.keys():
logger().info("%s%-40.40s %-30.30s %-4.4s %s" % logger().info("%s%-40.40s %-30.30s %-4.4s %s" %
@ -1761,7 +1781,8 @@ class ITUNES(DriverBase):
''' '''
''' '''
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)]) FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)])
N=0; result='' N = 0
result = ''
while src: while src:
s, src = src[:length], src[length:] s, src = src[:length], src[length:]
hexa = ' '.join(["%02X" % ord(x) for x in s]) hexa = ' '.join(["%02X" % ord(x) for x in s])
@ -1910,7 +1931,6 @@ class ITUNES(DriverBase):
if DEBUG: if DEBUG:
logger().error(" no Books playlist found") logger().error(" no Books playlist found")
attempts = 9 attempts = 9
while attempts: while attempts:
# Find book whose Album field = search['uuid'] # Find book whose Album field = search['uuid']
@ -2051,7 +2071,6 @@ class ITUNES(DriverBase):
return thumb_data return thumb_data
elif iswindows: elif iswindows:
if not book.Artwork.Count: if not book.Artwork.Count:
if DEBUG: if DEBUG:
@ -2133,14 +2152,13 @@ class ITUNES(DriverBase):
logger().error(" book_playlist not found") logger().error(" book_playlist not found")
for book in dev_books: for book in dev_books:
# This may need additional entries for international iTunes users
if book.kind() in self.Audiobooks: if book.kind() in self.Audiobooks:
if DEBUG: if DEBUG:
logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind()))
else: else:
if DEBUG: if DEBUG:
logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % logger().info(" %-40.40s %-30.30s %-40.40s [%s]" %
(book.name(), book.artist(), book.album(), book.kind())) (book.name(), book.artist(), book.composer(), book.kind()))
device_books.append(book) device_books.append(book)
if DEBUG: if DEBUG:
logger().info() logger().info()
@ -2167,13 +2185,12 @@ class ITUNES(DriverBase):
logger().info(" no Books playlist found") logger().info(" no Books playlist found")
for book in dev_books: for book in dev_books:
# This may need additional entries for international iTunes users
if book.KindAsString in self.Audiobooks: if book.KindAsString in self.Audiobooks:
if DEBUG: if DEBUG:
logger().info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) logger().info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString))
else: else:
if DEBUG: if DEBUG:
logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) logger().info(" %-40.40s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Composer, book.KindAsString))
device_books.append(book) device_books.append(book)
if DEBUG: if DEBUG:
logger().info() logger().info()
@ -2528,6 +2545,9 @@ class ITUNES(DriverBase):
Remove any iTunes orphans originally added by calibre Remove any iTunes orphans originally added by calibre
This occurs when the user deletes a book in iBooks while disconnected This occurs when the user deletes a book in iBooks while disconnected
''' '''
PURGE_ORPHANS = False
if PURGE_ORPHANS:
if DEBUG: if DEBUG:
logger().info(" %s._purge_orphans()" % self.__class__.__name__) logger().info(" %s._purge_orphans()" % self.__class__.__name__)
#self._dump_library_books(library_books) #self._dump_library_books(library_books)
@ -2539,7 +2559,8 @@ class ITUNES(DriverBase):
str(library_books[book].description()).startswith(self.description_prefix): str(library_books[book].description()).startswith(self.description_prefix):
if DEBUG: if DEBUG:
logger().info(" '%s' not found on iDevice, removing from iTunes" % book) logger().info(" '%s' not found on iDevice, removing from iTunes" % book)
btr = { 'title':library_books[book].name(), btr = {
'title': library_books[book].name(),
'author': library_books[book].artist(), 'author': library_books[book].artist(),
'lib_book': library_books[book]} 'lib_book': library_books[book]}
self._remove_from_iTunes(btr) self._remove_from_iTunes(btr)
@ -2548,12 +2569,14 @@ class ITUNES(DriverBase):
library_books[book].Description.startswith(self.description_prefix): library_books[book].Description.startswith(self.description_prefix):
if DEBUG: if DEBUG:
logger().info(" '%s' not found on iDevice, removing from iTunes" % book) logger().info(" '%s' not found on iDevice, removing from iTunes" % book)
btr = { 'title':library_books[book].Name, btr = {
'title': library_books[book].Name,
'author': library_books[book].Artist, 'author': library_books[book].Artist,
'lib_book': library_books[book]} 'lib_book': library_books[book]}
self._remove_from_iTunes(btr) self._remove_from_iTunes(btr)
else:
if DEBUG: if DEBUG:
logger().info() logger().info(" %s._purge_orphans(disabled)" % self.__class__.__name__)
def _remove_existing_copy(self, path, metadata): def _remove_existing_copy(self, path, metadata):
''' '''
@ -2565,17 +2588,11 @@ class ITUNES(DriverBase):
# Delete existing from Device|Books, add to self.update_list # Delete existing from Device|Books, add to self.update_list
# for deletion from booklist[0] during add_books_to_metadata # for deletion from booklist[0] during add_books_to_metadata
for book in self.cached_books: for book in self.cached_books:
if self.cached_books[book]['uuid'] == metadata.uuid or \ if (self.cached_books[book]['uuid'] == metadata.uuid or
(self.cached_books[book]['title'] == metadata.title and \ (self.cached_books[book]['title'] == metadata.title and
self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.cached_books[book]['author'] == metadata.author)):
self.update_list.append(self.cached_books[book]) self.update_list.append(self.cached_books[book])
if DEBUG:
logger().info( " deleting device book '%s'" % (metadata.title))
self._remove_from_device(self.cached_books[book]) self._remove_from_device(self.cached_books[book])
if DEBUG:
logger().info(" deleting library book '%s'" % metadata.title)
self._remove_from_iTunes(self.cached_books[book]) self._remove_from_iTunes(self.cached_books[book])
break break
else: else:
@ -2585,9 +2602,9 @@ class ITUNES(DriverBase):
# Delete existing from Library|Books, add to self.update_list # Delete existing from Library|Books, add to self.update_list
# for deletion from booklist[0] during add_books_to_metadata # for deletion from booklist[0] during add_books_to_metadata
for book in self.cached_books: for book in self.cached_books:
if self.cached_books[book]['uuid'] == metadata.uuid or \ if (self.cached_books[book]['uuid'] == metadata.uuid or
(self.cached_books[book]['title'] == metadata.title and \ (self.cached_books[book]['title'] == metadata.title and \
self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.cached_books[book]['author'] == metadata.author)):
self.update_list.append(self.cached_books[book]) self.update_list.append(self.cached_books[book])
if DEBUG: if DEBUG:
logger().info(" deleting library book '%s'" % metadata.title) logger().info(" deleting library book '%s'" % metadata.title)
@ -2668,7 +2685,8 @@ class ITUNES(DriverBase):
except: except:
# We get here if there was an error with .location().path # We get here if there was an error with .location().path
if DEBUG: if DEBUG:
logger().info(" '%s' not found in iTunes storage" % cached_book['title']) logger().info(" '%s' by %s not found in iTunes storage" %
(cached_book['title'], cached_book['author']))
# Delete the book from the iTunes database # Delete the book from the iTunes database
try: try:
@ -2953,7 +2971,6 @@ class ITUNES(DriverBase):
db_added.genre.set(tag) db_added.genre.set(tag)
break break
elif metadata_x.tags is not None: elif metadata_x.tags is not None:
if DEBUG: if DEBUG:
logger().info(" %susing Tag as Genre" % logger().info(" %susing Tag as Genre" %
@ -3147,6 +3164,7 @@ class ITUNES(DriverBase):
newmi = book newmi = book
return newmi return newmi
class ITUNES_ASYNC(ITUNES): class ITUNES_ASYNC(ITUNES):
''' '''
This subclass allows the user to interact directly with iTunes via a menu option This subclass allows the user to interact directly with iTunes via a menu option
@ -3441,6 +3459,7 @@ class ITUNES_ASYNC(ITUNES):
logger().info("%s.unmount_device()" % self.__class__.__name__) logger().info("%s.unmount_device()" % self.__class__.__name__)
self.connected = False self.connected = False
class BookList(list): class BookList(list):
''' '''
A list of books. Each Book object must have the fields: A list of books. Each Book object must have the fields:
@ -3493,6 +3512,7 @@ class BookList(list):
''' '''
return {} return {}
class Book(Metadata): class Book(Metadata):
''' '''
A simple class describing a book in the iTunes Books Library. A simple class describing a book in the iTunes Books Library.
@ -3500,9 +3520,9 @@ class Book(Metadata):
''' '''
def __init__(self, title, author): def __init__(self, title, author):
Metadata.__init__(self, title, authors=author.split(' & ')) Metadata.__init__(self, title, authors=author.split(' & '))
self.author = author
self.author_sort = author_to_author_sort(author) self.author_sort = author_to_author_sort(author)
@property @property
def title_sorter(self): def title_sorter(self):
return title_sort(self.title) return title_sort(self.title)

2
src/calibre/library/catalogs/csv_xml.py
View File

@ -12,6 +12,7 @@ from calibre.customize import CatalogPlugin
from calibre.library.catalogs import FIELDS from calibre.library.catalogs import FIELDS
from calibre.customize.conversion import DummyReporter from calibre.customize.conversion import DummyReporter
class CSV_XML(CatalogPlugin): class CSV_XML(CatalogPlugin):
'CSV/XML catalog generator' 'CSV/XML catalog generator'
@ -227,4 +228,3 @@ class CSV_XML(CatalogPlugin):
with open(path_to_output, 'w') as f: with open(path_to_output, 'w') as f:
f.write(etree.tostring(root, encoding='utf-8', f.write(etree.tostring(root, encoding='utf-8',
xml_declaration=True, pretty_print=True)) xml_declaration=True, pretty_print=True))

114
src/calibre/library/catalogs/epub_mobi_builder.py
View File

@ -25,6 +25,7 @@ from calibre.utils.icu import capitalize, collation_order, sort_key
from calibre.utils.magick.draw import thumbnail from calibre.utils.magick.draw import thumbnail
from calibre.utils.zipfile import ZipFile from calibre.utils.zipfile import ZipFile
class CatalogBuilder(object): class CatalogBuilder(object):
''' '''
Generates catalog source files from calibre database Generates catalog source files from calibre database
@ -98,7 +99,6 @@ class CatalogBuilder(object):
else: else:
return ' ' return ' '
def __init__(self, db, _opts, plugin, def __init__(self, db, _opts, plugin,
report_progress=DummyReporter(), report_progress=DummyReporter(),
stylesheet="content/stylesheet.css", stylesheet="content/stylesheet.css",
@ -120,11 +120,13 @@ class CatalogBuilder(object):
_opts.output_profile and _opts.output_profile and
_opts.output_profile.startswith("kindle")) else False _opts.output_profile.startswith("kindle")) else False
self.all_series = set()
self.authors = None self.authors = None
self.bookmarked_books = None self.bookmarked_books = None
self.bookmarked_books_by_date_read = None self.bookmarked_books_by_date_read = None
self.books_by_author = None self.books_by_author = None
self.books_by_date_range = None self.books_by_date_range = None
self.books_by_description = None
self.books_by_month = None self.books_by_month = None
self.books_by_series = None self.books_by_series = None
self.books_by_title = None self.books_by_title = None
@ -139,6 +141,7 @@ class CatalogBuilder(object):
if self.opts.generate_genres else None if self.opts.generate_genres else None
self.html_filelist_1 = [] self.html_filelist_1 = []
self.html_filelist_2 = [] self.html_filelist_2 = []
self.individual_authors = None
self.merge_comments_rule = dict(zip(['field', 'position', 'hr'], self.merge_comments_rule = dict(zip(['field', 'position', 'hr'],
_opts.merge_comments_rule.split(':'))) _opts.merge_comments_rule.split(':')))
self.ncx_soup = None self.ncx_soup = None
@ -154,6 +157,7 @@ class CatalogBuilder(object):
self.total_steps = 6.0 self.total_steps = 6.0
self.use_series_prefix_in_titles_section = False self.use_series_prefix_in_titles_section = False
self.dump_custom_fields()
self.books_to_catalog = self.fetch_books_to_catalog() self.books_to_catalog = self.fetch_books_to_catalog()
self.compute_total_steps() self.compute_total_steps()
self.calculate_thumbnail_dimensions() self.calculate_thumbnail_dimensions()
@ -447,7 +451,7 @@ class CatalogBuilder(object):
hits.remove(amp) hits.remove(amp)
for hit in hits: for hit in hits:
name = hit[1:-1] name = hit[1:-1]
if htmlentitydefs.name2codepoint.has_key(name): if htmlentitydefs.name2codepoint in name:
s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name]))
s = s.replace(amp, "&") s = s.replace(amp, "&")
return s return s
@ -586,7 +590,7 @@ class CatalogBuilder(object):
# Literal comparison for Tags field # Literal comparison for Tags field
if rule['field'].lower() == 'tags': if rule['field'].lower() == 'tags':
if rule['pattern'].lower() in map(unicode.lower, record['tags']): if rule['pattern'].lower() in map(unicode.lower, record['tags']):
if self.opts.verbose: if self.DEBUG and self.opts.verbose:
self.opts.log.info(" %s '%s' by %s (%s: Tags includes '%s')" % self.opts.log.info(" %s '%s' by %s (%s: Tags includes '%s')" %
(rule['prefix'], record['title'], (rule['prefix'], record['title'],
record['authors'][0], rule['name'], record['authors'][0], rule['name'],
@ -616,7 +620,7 @@ class CatalogBuilder(object):
try: try:
if re.search(rule['pattern'], unicode(field_contents), if re.search(rule['pattern'], unicode(field_contents),
re.IGNORECASE) is not None: re.IGNORECASE) is not None:
if self.opts.verbose: if self.DEBUG:
_log_prefix_rule_match_info(rule, record, field_contents) _log_prefix_rule_match_info(rule, record, field_contents)
return rule['prefix'] return rule['prefix']
except: except:
@ -624,12 +628,24 @@ class CatalogBuilder(object):
self.opts.log.error("pattern failed to compile: %s" % rule['pattern']) self.opts.log.error("pattern failed to compile: %s" % rule['pattern'])
pass pass
elif field_contents is None and rule['pattern'] == 'None': elif field_contents is None and rule['pattern'] == 'None':
if self.opts.verbose: if self.DEBUG:
_log_prefix_rule_match_info(rule, record, field_contents) _log_prefix_rule_match_info(rule, record, field_contents)
return rule['prefix'] return rule['prefix']
return None return None
def dump_custom_fields(self):
"""
Dump custom field mappings for debugging
"""
if self.opts.verbose:
self.opts.log.info(" Custom fields:")
all_custom_fields = self.db.custom_field_keys()
for cf in all_custom_fields:
self.opts.log.info(" %-20s %-20s %s" %
(cf, "'%s'" % self.db.metadata_for_field(cf)['name'],
self.db.metadata_for_field(cf)['datatype']))
def establish_equivalencies(self, item_list, key=None): def establish_equivalencies(self, item_list, key=None):
""" Return icu equivalent sort letter. """ Return icu equivalent sort letter.
@ -716,7 +732,8 @@ class CatalogBuilder(object):
Outputs: Outputs:
books_by_author: database, sorted by author books_by_author: database, sorted by author
authors: list of unique authors authors: list of book authors. Two credited authors are considered an
individual entity
error: author_sort mismatches error: author_sort mismatches
Return: Return:
@ -728,6 +745,12 @@ class CatalogBuilder(object):
books_by_author = list(self.books_to_catalog) books_by_author = list(self.books_to_catalog)
self.detect_author_sort_mismatches(books_by_author) self.detect_author_sort_mismatches(books_by_author)
# Assumes books_by_title already populated
# init books_by_description before relisting multiple authors
books_by_description = list(books_by_author) if self.opts.sort_descriptions_by_author \
else list(self.books_by_title)
if self.opts.cross_reference_authors: if self.opts.cross_reference_authors:
books_by_author = self.relist_multiple_authors(books_by_author) books_by_author = self.relist_multiple_authors(books_by_author)
@ -737,6 +760,9 @@ class CatalogBuilder(object):
asl = [i['author_sort'] for i in books_by_author] asl = [i['author_sort'] for i in books_by_author]
las = max(asl, key=len) las = max(asl, key=len)
self.books_by_description = sorted(books_by_description,
key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las))))
books_by_author = sorted(books_by_author, books_by_author = sorted(books_by_author,
key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las))))
@ -758,6 +784,7 @@ class CatalogBuilder(object):
current_author = authors[0] current_author = authors[0]
multiple_authors = False multiple_authors = False
unique_authors = [] unique_authors = []
individual_authors = set()
for (i, author) in enumerate(authors): for (i, author) in enumerate(authors):
if author != current_author: if author != current_author:
# Note that current_author and author are tuples: (friendly, sort) # Note that current_author and author are tuples: (friendly, sort)
@ -780,14 +807,23 @@ class CatalogBuilder(object):
unique_authors.append((current_author[0], icu_title(current_author[1]), unique_authors.append((current_author[0], icu_title(current_author[1]),
books_by_current_author)) books_by_current_author))
self.authors = list(unique_authors)
self.books_by_author = books_by_author
for ua in unique_authors:
for ia in ua[0].replace(' & ', ' & ').split(' & '):
individual_authors.add(ia)
self.individual_authors = list(individual_authors)
if self.DEBUG and self.opts.verbose: if self.DEBUG and self.opts.verbose:
self.opts.log.info("\nfetch_books_by_author(): %d unique authors" % len(unique_authors)) self.opts.log.info("\nfetch_books_by_author(): %d unique authors" % len(unique_authors))
for author in unique_authors: for author in unique_authors:
self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20],
author[2])).encode('utf-8')) author[2])).encode('utf-8'))
self.opts.log.info("\nfetch_books_by_author(): %d individual authors" % len(individual_authors))
for author in sorted(individual_authors):
self.opts.log.info("%s" % author)
self.authors = unique_authors
self.books_by_author = books_by_author
return True return True
def fetch_books_by_title(self): def fetch_books_by_title(self):
@ -869,6 +905,7 @@ class CatalogBuilder(object):
this_title['title'] = self.convert_html_entities(record['title']) this_title['title'] = self.convert_html_entities(record['title'])
if record['series']: if record['series']:
this_title['series'] = record['series'] this_title['series'] = record['series']
self.all_series.add(this_title['series'])
this_title['series_index'] = record['series_index'] this_title['series_index'] = record['series_index']
else: else:
this_title['series'] = None this_title['series'] = None
@ -1000,7 +1037,7 @@ class CatalogBuilder(object):
data = self.plugin.search_sort_db(self.db, self.opts) data = self.plugin.search_sort_db(self.db, self.opts)
data = self.process_exclusions(data) data = self.process_exclusions(data)
if self.opts.verbose and self.prefix_rules: if self.prefix_rules and self.DEBUG:
self.opts.log.info(" Added prefixes:") self.opts.log.info(" Added prefixes:")
# Populate this_title{} from data[{},{}] # Populate this_title{} from data[{},{}]
@ -1042,6 +1079,7 @@ class CatalogBuilder(object):
def initialize(self, save_template): def initialize(self, save_template):
self._save_template = save_template self._save_template = save_template
self.SUPPORTS_SUB_DIRS = True self.SUPPORTS_SUB_DIRS = True
def save_template(self): def save_template(self):
return self._save_template return self._save_template
@ -2070,7 +2108,6 @@ class CatalogBuilder(object):
len(genre[key]), len(genre[key]),
'titles' if len(genre[key]) > 1 else 'title')) 'titles' if len(genre[key]) > 1 else 'title'))
# Write the results # Write the results
# genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...] # genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...]
master_genre_list = [] master_genre_list = []
@ -2107,7 +2144,8 @@ class CatalogBuilder(object):
outfile) outfile)
tag_file = "content/Genre_%s.html" % genre tag_file = "content/Genre_%s.html" % genre
master_genre_list.append({'tag':genre, master_genre_list.append({
'tag': genre,
'file': tag_file, 'file': tag_file,
'authors': unique_authors, 'authors': unique_authors,
'books': genre_tag_set[genre], 'books': genre_tag_set[genre],
@ -2937,10 +2975,8 @@ class CatalogBuilder(object):
navPointTag.insert(1, contentTag) navPointTag.insert(1, contentTag)
else: else:
# Descriptions only # Descriptions only
sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \
else self.books_by_title
contentTag = Tag(soup, 'content') contentTag = Tag(soup, 'content')
contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) contentTag['src'] = "content/book_%d.html" % int(self.books_by_description[0]['id'])
navPointTag.insert(1, contentTag) navPointTag.insert(1, contentTag)
if self.generate_for_kindle_mobi: if self.generate_for_kindle_mobi:
@ -2970,9 +3006,6 @@ class CatalogBuilder(object):
self.update_progress_full_step(_("NCX for Descriptions")) self.update_progress_full_step(_("NCX for Descriptions"))
sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \
else self.books_by_title
# --- Construct the 'Descriptions' section --- # --- Construct the 'Descriptions' section ---
ncx_soup = self.ncx_soup ncx_soup = self.ncx_soup
if self.generate_for_kindle_mobi: if self.generate_for_kindle_mobi:
@ -2990,19 +3023,22 @@ class CatalogBuilder(object):
self.play_order += 1 self.play_order += 1
navLabelTag = Tag(ncx_soup, 'navLabel') navLabelTag = Tag(ncx_soup, 'navLabel')
textTag = Tag(ncx_soup, 'text') textTag = Tag(ncx_soup, 'text')
textTag.insert(0, NavigableString(tocTitle)) section_header = '%s [%d]' % (tocTitle, len(self.books_by_description))
if self.generate_for_kindle_mobi:
section_header = tocTitle
textTag.insert(0, NavigableString(section_header))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
nptc = 0 nptc = 0
navPointTag.insert(nptc, navLabelTag) navPointTag.insert(nptc, navLabelTag)
nptc += 1 nptc += 1
contentTag = Tag(ncx_soup, "content") contentTag = Tag(ncx_soup, "content")
contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) contentTag['src'] = "content/book_%d.html" % int(self.books_by_description[0]['id'])
navPointTag.insert(nptc, contentTag) navPointTag.insert(nptc, contentTag)
nptc += 1 nptc += 1
# Loop over the titles # Loop over the titles
for book in sort_descriptions_by: for book in self.books_by_description:
navPointVolumeTag = Tag(ncx_soup, 'navPoint') navPointVolumeTag = Tag(ncx_soup, 'navPoint')
if self.generate_for_kindle_mobi: if self.generate_for_kindle_mobi:
navPointVolumeTag['class'] = "article" navPointVolumeTag['class'] = "article"
@ -3119,7 +3155,10 @@ class CatalogBuilder(object):
self.play_order += 1 self.play_order += 1
navLabelTag = Tag(ncx_soup, 'navLabel') navLabelTag = Tag(ncx_soup, 'navLabel')
textTag = Tag(ncx_soup, 'text') textTag = Tag(ncx_soup, 'text')
textTag.insert(0, NavigableString(tocTitle)) section_header = '%s [%d]' % (tocTitle, len(self.all_series))
if self.generate_for_kindle_mobi:
section_header = tocTitle
textTag.insert(0, NavigableString(section_header))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
nptc = 0 nptc = 0
navPointTag.insert(nptc, navLabelTag) navPointTag.insert(nptc, navLabelTag)
@ -3247,7 +3286,10 @@ class CatalogBuilder(object):
self.play_order += 1 self.play_order += 1
navLabelTag = Tag(ncx_soup, 'navLabel') navLabelTag = Tag(ncx_soup, 'navLabel')
textTag = Tag(ncx_soup, 'text') textTag = Tag(ncx_soup, 'text')
textTag.insert(0, NavigableString(tocTitle)) section_header = '%s [%d]' % (tocTitle, len(self.books_by_title))
if self.generate_for_kindle_mobi:
section_header = tocTitle
textTag.insert(0, NavigableString(section_header))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
nptc = 0 nptc = 0
navPointTag.insert(nptc, navLabelTag) navPointTag.insert(nptc, navLabelTag)
@ -3377,7 +3419,10 @@ class CatalogBuilder(object):
self.play_order += 1 self.play_order += 1
navLabelTag = Tag(ncx_soup, 'navLabel') navLabelTag = Tag(ncx_soup, 'navLabel')
textTag = Tag(ncx_soup, 'text') textTag = Tag(ncx_soup, 'text')
textTag.insert(0, NavigableString('%s' % tocTitle)) section_header = '%s [%d]' % (tocTitle, len(self.individual_authors))
if self.generate_for_kindle_mobi:
section_header = tocTitle
textTag.insert(0, NavigableString(section_header))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
nptc = 0 nptc = 0
navPointTag.insert(nptc, navLabelTag) navPointTag.insert(nptc, navLabelTag)
@ -3430,7 +3475,7 @@ class CatalogBuilder(object):
fmt_string = _(u"Authors beginning with %s") fmt_string = _(u"Authors beginning with %s")
else: else:
fmt_string = _(u"Authors beginning with '%s'") fmt_string = _(u"Authors beginning with '%s'")
textTag.insert(0, NavigableString(fmt_string % (authors_by_letter[1]))) textTag.insert(0, NavigableString(fmt_string % authors_by_letter[1]))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
navPointByLetterTag.insert(0, navLabelTag) navPointByLetterTag.insert(0, navLabelTag)
contentTag = Tag(ncx_soup, 'content') contentTag = Tag(ncx_soup, 'content')
@ -3808,7 +3853,7 @@ class CatalogBuilder(object):
self.update_progress_full_step(_("NCX for Genres")) self.update_progress_full_step(_("NCX for Genres"))
if not len(self.genres): if not len(self.genres):
self.opts.log.warn(" No genres found in tags.\n" self.opts.log.warn(" No genres found\n"
" No Genre section added to Catalog") " No Genre section added to Catalog")
return return
@ -3830,8 +3875,10 @@ class CatalogBuilder(object):
self.play_order += 1 self.play_order += 1
navLabelTag = Tag(ncx_soup, 'navLabel') navLabelTag = Tag(ncx_soup, 'navLabel')
textTag = Tag(ncx_soup, 'text') textTag = Tag(ncx_soup, 'text')
# textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) section_header = '%s [%d]' % (tocTitle, len(self.genres))
textTag.insert(0, NavigableString('%s' % tocTitle)) if self.generate_for_kindle_mobi:
section_header = tocTitle
textTag.insert(0, NavigableString(section_header))
navLabelTag.insert(0, textTag) navLabelTag.insert(0, textTag)
nptc = 0 nptc = 0
navPointTag.insert(nptc, navLabelTag) navPointTag.insert(nptc, navLabelTag)
@ -3993,7 +4040,6 @@ class CatalogBuilder(object):
mtc += 1 mtc += 1
# Write the thumbnail images, descriptions to the manifest # Write the thumbnail images, descriptions to the manifest
sort_descriptions_by = []
if self.opts.generate_descriptions: if self.opts.generate_descriptions:
for thumb in self.thumbs: for thumb in self.thumbs:
itemTag = Tag(soup, "item") itemTag = Tag(soup, "item")
@ -4004,9 +4050,6 @@ class CatalogBuilder(object):
manifest.insert(mtc, itemTag) manifest.insert(mtc, itemTag)
mtc += 1 mtc += 1
# HTML files - add descriptions to manifest and spine
sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \
else self.books_by_title
# Add html_files to manifest and spine # Add html_files to manifest and spine
for file in self.html_filelist_1: for file in self.html_filelist_1:
@ -4060,7 +4103,8 @@ class CatalogBuilder(object):
spine.insert(stc, itemrefTag) spine.insert(stc, itemrefTag)
stc += 1 stc += 1
for book in sort_descriptions_by: if self.opts.generate_descriptions:
for book in self.books_by_description:
# manifest # manifest
itemTag = Tag(soup, "item") itemTag = Tag(soup, "item")
itemTag['href'] = "content/book_%d.html" % int(book['id']) itemTag['href'] = "content/book_%d.html" % int(book['id'])
@ -4286,7 +4330,8 @@ class CatalogBuilder(object):
f.write(thumb_data) f.write(thumb_data)
# Save thumb to archive # Save thumb to archive
if zf is not None: # Ensure that the read succeeded if zf is not None:
# Ensure that the read succeeded
# If we failed to open the zip file for reading, # If we failed to open the zip file for reading,
# we dont know if it contained the thumb or not # we dont know if it contained the thumb or not
zf = _open_archive('a') zf = _open_archive('a')
@ -4363,7 +4408,6 @@ class CatalogBuilder(object):
# Clear the book's cover property # Clear the book's cover property
title['cover'] = None title['cover'] = None
# Write thumb_width to the file, validating cache contents # Write thumb_width to the file, validating cache contents
# Allows detection of aborted catalog builds # Allows detection of aborted catalog builds
with ZipFile(self.thumbs_path, mode='a') as zfw: with ZipFile(self.thumbs_path, mode='a') as zfw:
@ -4853,5 +4897,3 @@ class CatalogBuilder(object):
outfile = open("%s/%s.ncx" % (self.catalog_path, self.opts.basename), 'w') outfile = open("%s/%s.ncx" % (self.catalog_path, self.opts.basename), 'w')
outfile.write(self.ncx_soup.prettify()) outfile.write(self.ncx_soup.prettify())