From 72fbd67c1764f1c1fc9e7dad00ffa144fab2bf04 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 14 May 2010 12:17:14 +0100 Subject: [PATCH] More testing of: 1) initial condition: cache does not exist on book 2) adding and removing books 3) subsequent conditions: cache exists In addition: 1) added metadata correction for books matched with something other than UUID. 2) Refactored changes to BookList, to move the additional methods into USMBS from Interface. 3) Made classmethods in USBMS into normal methods. --- src/calibre/devices/interface.py | 14 -------- src/calibre/devices/prs505/books.py | 39 ++++------------------ src/calibre/devices/prs505/driver.py | 4 ++- src/calibre/devices/usbms/books.py | 39 ++++++++++++++-------- src/calibre/devices/usbms/driver.py | 50 +++++++++++++--------------- src/calibre/gui2/device.py | 25 ++++++++++---- src/calibre/library/database2.py | 8 ++--- 7 files changed, 82 insertions(+), 97 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 6247e29e15..b38b62e20c 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -402,17 +402,3 @@ class BookList(list): ''' raise NotImplementedError() - def add_book(self, book, collections=None): - ''' - Add the book to the booklist. Intent is to maintain any device-internal - metadata - ''' - if book not in self: - self.append(book) - - def remove_book(self, book): - ''' - Remove a book from the booklist. Correct any device metadata at the - same time - ''' - self.remove(book) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index ba3605530e..855d8d5cd3 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -8,7 +8,7 @@ import xml.dom.minidom as dom from base64 import b64encode as encode -from calibre.devices.interface import BookList as _BookList +from calibre.devices.usbms.books import BookList as _BookList from calibre.devices import strftime as _strftime from calibre.devices.usbms.books import Book as _Book from calibre.devices.prs505 import MEDIA_XML @@ -31,36 +31,6 @@ def uuid(): def sortable_title(title): return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip() -class book_metadata_field(object): - """ Represents metadata stored as an attribute """ - def __init__(self, attr, formatter=None, setter=None): - self.attr = attr - self.formatter = formatter - self.setter = setter - - def __get__(self, obj, typ=None): - """ Return a string. String may be empty if self.attr is absent """ - return self.formatter(obj.elem.getAttribute(self.attr)) if \ - self.formatter else obj.elem.getAttribute(self.attr).strip() - - def __set__(self, obj, val): - """ Set the attribute """ - val = self.setter(val) if self.setter else val - if not isinstance(val, unicode): - val = unicode(val, 'utf8', 'replace') - obj.elem.setAttribute(self.attr, val) - - -class Book(_Book): - @dynamic_property - def db_id(self): - doc = '''The database id in the application database that this file corresponds to''' - def fget(self): - match = re.search(r'_(\d+)$', self.rpath.rpartition('.')[0]) - if match: - return int(match.group(1)) - return property(fget=fget, doc=doc) - class BookList(_BookList): def __init__(self, oncard, prefix): @@ -318,7 +288,12 @@ class BookList(_BookList): imap = {} for book, sony_id in zip(books, sony_ids): if book is not None: - imap[book.application_id] = sony_id + db_id = book.application_id + if db_id is None: + db_id = book.db_id + print 'here', db_id + if db_id is not None: + imap[book.application_id] = sony_id # filter the list, removing books not on device but on playlist books = [i for i in books if i is not None] # filter the order specification to the books we have diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 0b41894a18..1d403cb75d 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -13,6 +13,7 @@ import os import re from calibre.devices.usbms.driver import USBMS +from calibre.devices.usbms.books import Book from calibre.devices.prs505.books import BookList, fix_ids from calibre.devices.prs505 import MEDIA_XML from calibre.devices.prs505 import CACHE_XML @@ -59,8 +60,9 @@ class PRS505(USBMS): METADATA_CACHE = "database/cache/metadata.calibre" def initialize(self): - USBMS.initialize(self) + USBMS.initialize(self) # Must be first, so _class vars are set right self.booklist_class = BookList + self.book_class = Book def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index bc6003de27..ce74db6f54 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -37,12 +37,8 @@ class Book(MetaInformation): else: self.lpath = lpath self.mime = mime_type_ext(path_to_ext(lpath)) - self.size = os.stat(self.path).st_size if size == None else size - self.db_id = None - try: - self.datetime = time.gmtime(os.path.getctime(self.path)) - except ValueError: - self.datetime = time.gmtime() + self.size = None # will be set later + self.datetime = time.gmtime() if other: self.smart_update(other) @@ -70,6 +66,16 @@ class Book(MetaInformation): return spath == opath + @dynamic_property + def db_id(self): + doc = '''The database id in the application database that this file corresponds to''' + def fget(self): + match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0]) + if match: + return int(match.group(1)) + return None + return property(fget=fget, doc=doc) + @dynamic_property def title_sorter(self): doc = '''String to sort the title. If absent, title is returned''' @@ -81,13 +87,6 @@ class Book(MetaInformation): def thumbnail(self): return None -# def __str__(self): -# ''' -# Return a utf-8 encoded string with title author and path information -# ''' -# return self.title.encode('utf-8') + " by " + \ -# self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') - def smart_update(self, other): ''' Merge the information in C{other} into self. In case of conflicts, the information @@ -115,3 +114,17 @@ class BookList(_BookList): def set_tags(self, book, tags): book.tags = tags + def add_book(self, book, collections=None): + ''' + Add the book to the booklist. Intent is to maintain any device-internal + metadata + ''' + if book not in self: + self.append(book) + + def remove_book(self, book): + ''' + Remove a book from the booklist. Correct any device metadata at the + same time + ''' + self.remove(book) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 1a5b7461ed..361f7ea1bf 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -34,6 +34,7 @@ class USBMS(CLI, Device): def initialize(self): Device.initialize(self) self.booklist_class = BookList + self.book_class = Book def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) @@ -52,7 +53,9 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Getting list of books on device...')) return [] - prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix + prefix = self._card_a_prefix if oncard == 'carda' else \ + self._card_b_prefix if oncard == 'cardb' \ + else self._main_prefix metadata = self.booklist_class(oncard, prefix) ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ @@ -61,7 +64,6 @@ class USBMS(CLI, Device): bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE, self.booklist_class(oncard, prefix)) - # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): @@ -77,10 +79,10 @@ class USBMS(CLI, Device): lpath = lpath[len(os.sep):] idx = bl_cache.get(lpath.replace('\\', '/'), None) if idx is not None: - item, changed = self.__class__.update_metadata_item(bl[idx]) + item, changed = self.update_metadata_item(bl[idx]) self.count_found_in_bl += 1 else: - item = self.__class__.book_from_path(prefix, lpath) + item = self.book_from_path(prefix, lpath) changed = True metadata.append(item) except: # Probably a filename encoding error @@ -175,7 +177,10 @@ class USBMS(CLI, Device): lpath = path.partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - book = Book(prefix, lpath, other=info) + book = self.book_class(prefix, lpath, other=info) + if book.size is None: + book.size = os.stat(path).st_size + opts = self.settings() collections = opts.extra_customization.split(',') if opts.extra_customization else [] booklists[blist].add_book(book, collections, *location[1:-1]) @@ -229,19 +234,14 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) - @classmethod - def parse_metadata_cache(cls, prefix, name, bl): + def parse_metadata_cache(self, prefix, name, bl): js = [] need_sync = False try: with open(os.path.join(prefix, name), 'rb') as f: js = json.load(f, encoding='utf-8') for item in js: - lpath = item.get('lpath', None) - if not lpath or not os.path.exists(os.path.join(prefix, lpath)): - need_sync = True - continue - book = Book(prefix, lpath) + book = self.book_class(prefix, item.get('lpath', None)) for key in item.keys(): setattr(book, key, item[key]) bl.append(book) @@ -249,35 +249,33 @@ class USBMS(CLI, Device): import traceback traceback.print_exc() bl = [] + need_sync = True return bl, need_sync - @classmethod - def update_metadata_item(cls, item): + def update_metadata_item(self, item): changed = False size = os.stat(item.path).st_size if size != item.size: changed = True - mi = cls.metadata_from_path(item.path) + mi = self.metadata_from_path(item.path) item.smart_update(mi) + item.size = size return item, changed - @classmethod - def metadata_from_path(cls, path): - return cls.metadata_from_formats([path]) + def metadata_from_path(self, path): + return self.metadata_from_formats([path]) - @classmethod - def metadata_from_formats(cls, fmts): + def metadata_from_formats(self, fmts): from calibre.ebooks.metadata.meta import metadata_from_formats from calibre.customize.ui import quick_metadata with quick_metadata: return metadata_from_formats(fmts) - @classmethod - def book_from_path(cls, prefix, path): + def book_from_path(self, prefix, path): from calibre.ebooks.metadata import MetaInformation - if cls.settings().read_metadata or cls.MUST_READ_METADATA: - mi = cls.metadata_from_path(os.path.join(prefix, path)) + if self.settings().read_metadata or self.MUST_READ_METADATA: + mi = self.metadata_from_path(os.path.join(prefix, path)) else: from calibre.ebooks.metadata.meta import metadata_from_filename mi = metadata_from_filename(os.path.basename(path), @@ -286,6 +284,6 @@ class USBMS(CLI, Device): if mi is None: mi = MetaInformation(os.path.splitext(os.path.basename(path))[0], [_('Unknown')]) - - book = Book(prefix, path, other=mi) + mi.size = os.stat(os.path.join(prefix, path)).st_size + book = self.book_class(prefix, path, other=mi) return book diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 828756e2c8..a9f69bbca5 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -523,7 +523,8 @@ class DeviceGUI(object): d = ChooseFormatDialog(self, _('Choose format to send to device'), self.device_manager.device.settings().format_map) d.exec_() - fmt = d.format().lower() + if d.format(): + fmt = d.format().lower() dest, sub_dest = dest.split(':') if dest in ('main', 'carda', 'cardb'): if not self.device_connected or not self.device_manager: @@ -998,7 +999,7 @@ class DeviceGUI(object): if changed: self.library_view.model().refresh_ids(list(changed)) - def book_on_device(self, index, format=None, reset=False): + def book_on_device(self, id, format=None, reset=False): loc = [None, None, None] if reset: @@ -1030,7 +1031,7 @@ class DeviceGUI(object): if uuid is not None: self.book_db_uuid_cache[i].add(uuid) - mi = self.library_view.model().db.get_metadata(index, index_is_id=True) + mi = self.library_view.model().db.get_metadata(id, index_is_id=True) for i, l in enumerate(self.booklists()): if mi.uuid in self.book_db_uuid_cache[i]: loc[i] = True @@ -1038,7 +1039,7 @@ class DeviceGUI(object): db_title = re.sub('(?u)\W|[_]', '', mi.title.lower()) cache = self.book_db_title_cache[i].get(db_title, None) if cache: - if index in cache['db_ids']: + if id in cache['db_ids']: loc[i] = True break if mi.authors and \ @@ -1057,11 +1058,11 @@ class DeviceGUI(object): mi = self.library_view.model().db.get_metadata(idx, index_is_id=False) title = re.sub('(?u)\W|[_]', '', mi.title.lower()) if title not in self.db_book_title_cache: - self.db_book_title_cache[title] = {'authors':set(), 'db_ids':set()} + self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}} authors = authors_to_string(mi.authors).lower() if mi.authors else '' authors = re.sub('(?u)\W|[_]', '', authors) - self.db_book_title_cache[title]['authors'].add(authors) - self.db_book_title_cache[title]['db_ids'].add(mi.application_id) + self.db_book_title_cache[title]['authors'][authors] = mi + self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi self.db_book_uuid_cache.add(mi.uuid) # Now iterate through all the books on the device, setting the in_library field @@ -1069,6 +1070,7 @@ class DeviceGUI(object): # is really the db key, but as this can accidentally match across libraries we # also verify the title. The db_id exists on Sony devices. Fallback is title # and author match + resend_metadata = False for booklist in booklists: for book in booklist: if getattr(book, 'uuid', None) in self.db_book_uuid_cache: @@ -1082,11 +1084,20 @@ class DeviceGUI(object): if d is not None: if getattr(book, 'application_id', None) in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.application_id]) + resend_metadata = True continue if book.db_id in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.db_id]) + resend_metadata = True continue book_authors = authors_to_string(book.authors).lower() if book.authors else '' book_authors = re.sub('(?u)\W|[_]', '', book_authors) if book_authors in d['authors']: book.in_library = True + book.smart_update(d['authors'][book_authors]) + resend_metadata = True + if resend_metadata: + # Correcting metadata cache on device. + self.device_manager.sync_booklists(None, booklists) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index fd59503eed..b0f2d3cb39 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -470,14 +470,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): im = PILImage.open(f) im.convert('RGB').save(path, 'JPEG') - def book_on_device(self, index): + def book_on_device(self, id): if callable(self.book_on_device_func): - return self.book_on_device_func(index) + return self.book_on_device_func(id) return None - def book_on_device_string(self, index): + def book_on_device_string(self, id): loc = [] - on = self.book_on_device(index) + on = self.book_on_device(id) if on is not None: m, a, b = on if m is not None: