From 9b7815acf1bca13063cd70c4ceb9bedfd0410b4f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 12 May 2010 15:40:01 +0100 Subject: [PATCH 01/20] Part way through normalization of path names in caching, and also performance improvements to ondevice matching --- src/calibre/devices/usbms/driver.py | 12 ++++++---- src/calibre/gui2/device.py | 36 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 0d3779a309..3d65dfba35 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,6 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' +import posixpath import os import re import json @@ -61,7 +62,7 @@ class USBMS(CLI, Device): # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): - bl_cache[b.path] = idx + bl_cache[b.lpath] = idx self.count_found_in_bl = 0 def update_booklist(filename, path, prefix): @@ -71,9 +72,9 @@ class USBMS(CLI, Device): lpath = os.path.join(path, filename).partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - p = os.path.join(prefix, lpath) - if p in bl_cache: - item, changed = self.__class__.update_metadata_item(bl[bl_cache[p]]) + idx = bl_cache.get(lpath.replace('\\', '/'), None) + if idx is not None: + item, changed = self.__class__.update_metadata_item(bl[idx]) self.count_found_in_bl += 1 else: item = self.__class__.book_from_path(prefix, lpath) @@ -109,6 +110,7 @@ class USBMS(CLI, Device): # find on the device. If need_sync is True then there were either items # on the device that were not in bl or some of the items were changed. if self.count_found_in_bl != len(bl) or need_sync: + print 'resync' if oncard == 'cardb': self.sync_booklists((None, None, metadata)) elif oncard == 'carda': @@ -173,7 +175,7 @@ class USBMS(CLI, Device): lpath = path.partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - + lpath = lpath.replace('\\', '/') book = Book(prefix, lpath, other=info) if book not in booklists[blist]: diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index f890515aa5..9511e1c752 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1043,29 +1043,37 @@ class DeviceGUI(object): def set_books_in_library(self, booklists, reset=False): if reset: - # First build a self.book_in_library_cache of the library, so the search isn't On**2 - self.book_in_library_cache = {} - for id, title in self.library_view.model().db.all_titles(): - title = re.sub('(?u)\W|[_]', '', title.lower()) - if title not in self.book_in_library_cache: - self.book_in_library_cache[title] = {'authors':set(), 'db_ids':set(), 'uuids':set()} - au = self.library_view.model().db.authors(id, index_is_id=True) - authors = au.lower() if au else '' + # First build a cache of the library, so the search isn't On**2 + self.db_book_title_cache = {} + self.db_book_uuid_cache = set() + for idx in range(self.library_view.model().db.count()): + mi = self.library_view.model().db.get_metadata(idx, index_is_id=False) + title = re.sub('(?u)\W|[_]', '', mi.title.lower()) + if title not in self.db_book_title_cache: + self.db_book_title_cache[title] = {'authors':set(), 'db_ids':set()} + authors = authors_to_string(mi.authors).lower() if mi.authors else '' authors = re.sub('(?u)\W|[_]', '', authors) - self.book_in_library_cache[title]['authors'].add(authors) - self.book_in_library_cache[title]['db_ids'].add(id) - self.book_in_library_cache[title]['uuids'].add(self.library_view.model().db.uuid(id, index_is_id=True)) + self.db_book_title_cache[title]['authors'].add(authors) + self.db_book_title_cache[title]['db_ids'].add(id) + self.db_book_uuid_cache.add(mi.uuid) # Now iterate through all the books on the device, setting the in_library field + # Fastest and most accurate key is the uuid. Second is the application_id, which + # is really the db key, but as this can accidentally match across libraries we + # also verify the title. The db_id exists on Sony devices. Fallback is title + # and author match for booklist in booklists: for book in booklist: + if getattr(book, 'uuid', None) in self.db_book_uuid_cache: + self.book_in_library = True + continue + book_title = book.title.lower() if book.title else '' book_title = re.sub('(?u)\W|[_]', '', book_title) book.in_library = False - d = self.book_in_library_cache.get(book_title, None) + d = self.db_book_title_cache.get(book_title, None) if d is not None: - if getattr(book, 'uuid', None) in d['uuids'] or \ - getattr(book, 'application_id', None) in d['db_ids']: + if getattr(book, 'application_id', None) in d['db_ids']: book.in_library = True continue if book.db_id in d['db_ids']: From cd6c46dba5bea1bc8a6de2aa5f29dcaee9020ef0 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 13 May 2010 09:56:41 +0100 Subject: [PATCH 02/20] Normalized paths and performance improvements done --- src/calibre/devices/usbms/books.py | 2 ++ src/calibre/devices/usbms/driver.py | 1 - src/calibre/gui2/device.py | 51 +++++++++++++++-------------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index eca9a27096..990b335a6d 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -31,6 +31,8 @@ class Book(MetaInformation): MetaInformation.__init__(self, '') self.path = os.path.join(prefix, lpath) + if os.sep == '\\': + self.path = self.path.replace('/', '\\') self.lpath = lpath self.mime = mime_type_ext(path_to_ext(lpath)) self.size = os.stat(self.path).st_size if size == None else size diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 3d65dfba35..63d28f5457 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,7 +10,6 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import posixpath import os import re import json diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 9511e1c752..df355f0ef5 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -999,44 +999,47 @@ class DeviceGUI(object): loc = [None, None, None] if reset: - self.book_on_device_cache = None + self.book_db_title_cache = None + self.book_db_uuid_cache = None return - if self.book_on_device_cache is None: - self.book_on_device_cache = [] + if self.book_db_title_cache is None: + self.book_db_title_cache = [] + self.book_db_uuid_cache = [] for i, l in enumerate(self.booklists()): - self.book_on_device_cache.append({}) + self.book_db_title_cache.append({}) + self.book_db_uuid_cache.append(set()) for book in l: book_title = book.title.lower() if book.title else '' book_title = re.sub('(?u)\W|[_]', '', book_title) - if book_title not in self.book_on_device_cache[i]: - self.book_on_device_cache[i][book_title] = \ + if book_title not in self.book_db_title_cache[i]: + self.book_db_title_cache[i][book_title] = \ {'authors':set(), 'db_ids':set(), 'uuids':set()} book_authors = authors_to_string(book.authors).lower() book_authors = re.sub('(?u)\W|[_]', '', book_authors) - self.book_on_device_cache[i][book_title]['authors'].add(book_authors) + self.book_db_title_cache[i][book_title]['authors'].add(book_authors) id = getattr(book, 'application_id', None) if id is None: id = book.db_id if id is not None: - self.book_on_device_cache[i][book_title]['db_ids'].add(id) + self.book_db_title_cache[i][book_title]['db_ids'].add(id) uuid = getattr(book, 'uuid', None) - if uuid is None: - self.book_on_device_cache[i][book_title]['uuids'].add(uuid) + if uuid is not None: + self.book_db_uuid_cache[i].add(uuid) - db = self.library_view.model().db - db_title = db.title(index, index_is_id=True).lower() - db_title = re.sub('(?u)\W|[_]', '', db_title) - db_authors = db.authors(index, index_is_id=True) - db_authors = db_authors.lower() if db_authors else '' - db_authors = re.sub('(?u)\W|[_]', '', db_authors) - db_uuid = db.uuid(index, index_is_id=True) + mi = self.library_view.model().db.get_metadata(index, index_is_id=True) for i, l in enumerate(self.booklists()): - d = self.book_on_device_cache[i].get(db_title, None) - if d: - if db_uuid in d['uuids'] or \ - index in d['db_ids'] or \ - db_authors in d['authors']: + if mi.uuid in self.book_db_uuid_cache[i]: + loc[i] = True + continue + db_title = re.sub('(?u)\W|[_]', '', mi.title.lower()) + cache = self.book_db_title_cache[i].get(db_title, None) + if cache: + if index in cache['db_ids']: + loc[i] = True + break + if mi.authors and \ + re.sub('(?u)\W|[_]', '', mi.authors.lower()) in cache['authors']: loc[i] = True break return loc @@ -1054,7 +1057,7 @@ class DeviceGUI(object): authors = authors_to_string(mi.authors).lower() if mi.authors else '' authors = re.sub('(?u)\W|[_]', '', authors) self.db_book_title_cache[title]['authors'].add(authors) - self.db_book_title_cache[title]['db_ids'].add(id) + self.db_book_title_cache[title]['db_ids'].add(mi.application_id) self.db_book_uuid_cache.add(mi.uuid) # Now iterate through all the books on the device, setting the in_library field @@ -1065,7 +1068,7 @@ class DeviceGUI(object): for booklist in booklists: for book in booklist: if getattr(book, 'uuid', None) in self.db_book_uuid_cache: - self.book_in_library = True + book.in_library = True continue book_title = book.title.lower() if book.title else '' From d1c040d5464798e79c865fa8f355eac6601c8d0d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 13 May 2010 21:57:54 +0100 Subject: [PATCH 03/20] Working JSON metadata along side Sony metadata --- src/calibre/devices/interface.py | 17 ++ src/calibre/devices/prs505/__init__.py | 4 + src/calibre/devices/prs505/books.py | 205 ++++++++---------------- src/calibre/devices/prs505/driver.py | 157 ++---------------- src/calibre/devices/usbms/books.py | 4 +- src/calibre/devices/usbms/driver.py | 37 +++-- src/calibre/ebooks/metadata/__init__.py | 8 +- src/calibre/gui2/device.py | 14 +- src/calibre/gui2/library.py | 15 +- 9 files changed, 142 insertions(+), 319 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 98421959cc..6247e29e15 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -387,6 +387,9 @@ class BookList(list): __getslice__ = None __setslice__ = None + def __init__(self, oncard, prefix): + pass + def supports_tags(self): ''' Return True if the the device supports tags (collections) for this book list. ''' raise NotImplementedError() @@ -399,3 +402,17 @@ class BookList(list): ''' raise NotImplementedError() + def add_book(self, book, collections=None): + ''' + Add the book to the booklist. Intent is to maintain any device-internal + metadata + ''' + if book not in self: + self.append(book) + + def remove_book(self, book): + ''' + Remove a book from the booklist. Correct any device metadata at the + same time + ''' + self.remove(book) diff --git a/src/calibre/devices/prs505/__init__.py b/src/calibre/devices/prs505/__init__.py index f832dbb7fc..20f3b8d49b 100644 --- a/src/calibre/devices/prs505/__init__.py +++ b/src/calibre/devices/prs505/__init__.py @@ -1,2 +1,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' + +MEDIA_XML = 'database/cache/media.xml' + +CACHE_XML = 'Sony Reader/database/cache.xml' diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 66f24b97a0..82bc977bcd 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -5,13 +5,14 @@ __copyright__ = '2008, Kovid Goyal ' import re, time, functools from uuid import uuid4 as _uuid import xml.dom.minidom as dom -from base64 import b64decode as decode from base64 import b64encode as encode from calibre.devices.interface import BookList as _BookList from calibre.devices import strftime as _strftime -from calibre.devices import strptime +from calibre.devices.usbms.books import Book as _Book +from calibre.devices.prs505 import MEDIA_XML +from calibre.devices.prs505 import CACHE_XML strftime = functools.partial(_strftime, zone=time.gmtime) @@ -50,62 +51,7 @@ class book_metadata_field(object): obj.elem.setAttribute(self.attr, val) -class Book(object): - """ Provides a view onto the XML element that represents a book """ - - title = book_metadata_field("title") - authors = book_metadata_field("author", \ - formatter=lambda x: [x if x and x.strip() else _('Unknown')]) - mime = book_metadata_field("mime") - rpath = book_metadata_field("path") - id = book_metadata_field("id", formatter=int) - sourceid = book_metadata_field("sourceid", formatter=int) - size = book_metadata_field("size", formatter=lambda x : int(float(x))) - # When setting this attribute you must use an epoch - datetime = book_metadata_field("date", formatter=strptime, setter=strftime) - - @dynamic_property - def title_sorter(self): - doc = '''String to sort the title. If absent, title is returned''' - def fget(self): - src = self.elem.getAttribute('titleSorter').strip() - if not src: - src = self.title - return src - def fset(self, val): - self.elem.setAttribute('titleSorter', sortable_title(unicode(val))) - return property(doc=doc, fget=fget, fset=fset) - - @dynamic_property - def thumbnail(self): - doc = \ - """ - The thumbnail. Should be a height 68 image. - Setting is not supported. - """ - def fget(self): - th = self.elem.getElementsByTagName(self.prefix + "thumbnail") - if not len(th): - th = self.elem.getElementsByTagName("cache:thumbnail") - if len(th): - for n in th[0].childNodes: - if n.nodeType == n.ELEMENT_NODE: - th = n - break - rc = "" - for node in th.childNodes: - if node.nodeType == node.TEXT_NODE: - rc += node.data - return decode(rc) - return property(fget=fget, doc=doc) - - @dynamic_property - def path(self): - doc = """ Absolute path to book on device. Setting not supported. """ - def fget(self): - return self.mountpath + self.rpath - return property(fget=fget, doc=doc) - +class Book(_Book): @dynamic_property def db_id(self): doc = '''The database id in the application database that this file corresponds to''' @@ -115,42 +61,26 @@ class Book(object): return int(match.group(1)) return property(fget=fget, doc=doc) - def __init__(self, node, mountpath, tags, prefix=""): - self.elem = node - self.prefix = prefix - self.tags = tags - self.mountpath = mountpath - - def __str__(self): - """ Return a utf-8 encoded string with title author and path information """ - return self.title.encode('utf-8') + " by " + \ - self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') - - class BookList(_BookList): - def __init__(self, xml_file, mountpath, report_progress=None): - _BookList.__init__(self) + def __init__(self, oncard, prefix): + _BookList.__init__(self, oncard, prefix) + if prefix is None: + return + db = CACHE_XML if oncard else MEDIA_XML + xml_file = open(prefix + db, 'rb') xml_file.seek(0) self.document = dom.parse(xml_file) self.root_element = self.document.documentElement - self.mountpath = mountpath + self.mountpath = prefix records = self.root_element.getElementsByTagName('records') - self.tag_order = {} if records: self.prefix = 'xs1:' self.root_element = records[0] else: self.prefix = '' - - nodes = self.root_element.childNodes - for i, book in enumerate(nodes): - if report_progress: - report_progress((i+1) / float(len(nodes)), _('Getting list of books on device...')) - if hasattr(book, 'tagName') and book.tagName.endswith('text'): - tags = [i.getAttribute('title') for i in self.get_playlists(book.getAttribute('id'))] - self.append(Book(book, mountpath, tags, prefix=self.prefix)) + self.tag_order = {} def max_id(self): max = 0 @@ -180,32 +110,32 @@ class BookList(_BookList): return child return None - def add_book(self, mi, name, collections, size, ctime): + def add_book(self, book, collections): + if book in self: + return """ Add a node into the DOM tree, representing a book """ - book = self.book_by_path(name) - if book is not None: - self.remove_book(name) - node = self.document.createElement(self.prefix + "text") - mime = MIME_MAP.get(name.rpartition('.')[-1].lower(), MIME_MAP['epub']) + mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) cid = self.max_id()+1 + book.sony_id = cid + self.append(book) try: sourceid = str(self[0].sourceid) if len(self) else '1' except: sourceid = '1' attrs = { - "title" : mi.title, - 'titleSorter' : sortable_title(mi.title), - "author" : mi.format_authors() if mi.format_authors() else _('Unknown'), + "title" : book.title, + 'titleSorter' : sortable_title(book.title), + "author" : book.format_authors() if book.format_authors() else _('Unknown'), "page":"0", "part":"0", "scale":"0", \ "sourceid":sourceid, "id":str(cid), "date":"", \ - "mime":mime, "path":name, "size":str(size) + "mime":mime, "path":book.lpath, "size":str(book.size) } for attr in attrs.keys(): node.setAttributeNode(self.document.createAttribute(attr)) node.setAttribute(attr, attrs[attr]) try: - w, h, data = mi.thumbnail + w, h, data = book.thumbnail except: w, h, data = None, None, None @@ -218,14 +148,11 @@ class BookList(_BookList): th.appendChild(jpeg) node.appendChild(th) self.root_element.appendChild(node) - book = Book(node, self.mountpath, [], prefix=self.prefix) - book.datetime = ctime - self.append(book) tags = [] for item in collections: item = item.strip() - mitem = getattr(mi, item, None) + mitem = getattr(book, item, None) titems = [] if mitem: if isinstance(mitem, list): @@ -241,37 +168,34 @@ class BookList(_BookList): tags.extend(titems) if tags: tags = list(set(tags)) - if hasattr(mi, 'tag_order'): - self.tag_order.update(mi.tag_order) - self.set_tags(book, tags) + if hasattr(book, 'tag_order'): + self.tag_order.update(book.tag_order) + self.set_playlists(cid, tags) - def _delete_book(self, node): + def _delete_node(self, node): nid = node.getAttribute('id') self.remove_from_playlists(nid) node.parentNode.removeChild(node) node.unlink() - def delete_book(self, cid): + def delete_node(self, lpath): ''' - Remove DOM node corresponding to book with C{id == cid}. + Remove DOM node corresponding to book with lpath. Also remove book from any collections it is part of. ''' - for book in self: - if str(book.id) == str(cid): - self.remove(book) - self._delete_book(book.elem) - break + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + if child.getAttribute('path') == lpath: + self._delete_node(child) + break - def remove_book(self, path): + def remove_book(self, book): ''' Remove DOM node corresponding to book with C{path == path}. Also remove book from any collections it is part of. ''' - for book in self: - if path.endswith(book.rpath): - self.remove(book) - self._delete_book(book.elem) - break + self.remove(book) + self.delete_node(book.lpath) def playlists(self): ans = [] @@ -343,11 +267,6 @@ class BookList(_BookList): pli.parentNode.removeChild(pli) pli.unlink() - def set_tags(self, book, tags): - tags = [t for t in tags if t] - book.tags = tags - self.set_playlists(book.id, tags) - def set_playlists(self, id, collections): self.remove_from_playlists(id) for collection in set(collections): @@ -358,15 +277,6 @@ class BookList(_BookList): item.setAttribute('id', str(id)) coll.appendChild(item) - def get_playlists(self, bookid): - ans = [] - for pl in self.playlists(): - for item in pl.childNodes: - if hasattr(item, 'tagName') and item.tagName.endswith('item'): - if item.getAttribute('id') == str(bookid): - ans.append(pl) - return ans - def next_id(self): return self.document.documentElement.getAttribute('nextID') @@ -378,27 +288,41 @@ class BookList(_BookList): src = self.document.toxml('utf-8') + '\n' stream.write(src.replace("'", ''')) - def book_by_id(self, id): - for book in self: - if str(book.id) == str(id): - return book - def reorder_playlists(self): + sony_id_cache = {} + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + + books_lpath_cache = {} + for book in self: + books_lpath_cache[book.lpath] = book + for title in self.tag_order.keys(): pl = self.playlist_by_title(title) if not pl: continue - db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')] - pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids] + # make a list of the ids + sony_ids = [id.getAttribute('id') \ + for id in pl.childNodes if hasattr(id, 'getAttribute')] + # convert IDs in playlist to a list of lpaths + sony_paths = [sony_id_cache[id] for id in sony_ids] + # create list of books containing lpaths + books = [books_lpath_cache.get(p, None) for p in sony_paths] + # create dict of db_id -> sony_id imap = {} - for i, j in zip(pl_book_ids, db_ids): - imap[i] = j - pl_book_ids = [i for i in pl_book_ids if i is not None] - ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids] + for book, sony_id in zip(books, sony_ids): + if book is not None: + imap[book.application_id] = sony_id + # filter the list, removing books not on device but on playlist + books = [i for i in books if i is not None] + # filter the order specification to the books we have + ordered_ids = [db_id for db_id in self.tag_order[title] if db_id in imap] + # rewrite the playlist in the correct order if len(ordered_ids) < len(pl.childNodes): continue - children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] + children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] for child in children: pl.removeChild(child) child.unlink() @@ -439,7 +363,6 @@ def fix_ids(main, carda, cardb): except KeyError: item.parentNode.removeChild(item) item.unlink() - db.reorder_playlists() regen_ids(main) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index f4fc4b0d29..3e1ee67faa 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -11,15 +11,14 @@ Device driver for the SONY PRS-505 import os import re -import time -from itertools import cycle -from calibre.devices.usbms.cli import CLI -from calibre.devices.usbms.device import Device +from calibre.devices.usbms.driver import USBMS from calibre.devices.prs505.books import BookList, fix_ids +from calibre.devices.prs505 import MEDIA_XML +from calibre.devices.prs505 import CACHE_XML from calibre import __appname__ -class PRS505(CLI, Device): +class PRS505(USBMS): name = 'PRS-300/505 Device Interface' gui_name = 'SONY Reader' @@ -46,9 +45,6 @@ class PRS505(CLI, Device): MAIN_MEMORY_VOLUME_LABEL = 'Sony Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'Sony Reader Storage Card' - MEDIA_XML = 'database/cache/media.xml' - CACHE_XML = 'Sony Reader/database/cache.xml' - CARD_PATH_PREFIX = __appname__ SUPPORTS_SUB_DIRS = True @@ -60,67 +56,18 @@ class PRS505(CLI, Device): 'series, tags, authors' EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags']) + METADATA_CACHE = "database/cache/metadata.calibre" + + def initialize(self): + USBMS.initialize(self) + self.booklist_class = BookList + def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id - def open(self): - self.report_progress = lambda x, y: x - Device.open(self) - - def write_cache(prefix): - try: - cachep = os.path.join(prefix, *(self.CACHE_XML.split('/'))) - if not os.path.exists(cachep): - dname = os.path.dirname(cachep) - if not os.path.exists(dname): - try: - os.makedirs(dname, mode=0777) - except: - time.sleep(5) - os.makedirs(dname, mode=0777) - with open(cachep, 'wb') as f: - f.write(u''' - - - '''.encode('utf8')) - return True - except: - import traceback - traceback.print_exc() - return False - - if self._card_a_prefix is not None: - if not write_cache(self._card_a_prefix): - self._card_a_prefix = None - if self._card_b_prefix is not None: - if not write_cache(self._card_b_prefix): - self._card_b_prefix = None - def get_device_information(self, end_session=True): return (self.gui_name, '', '', '') - def books(self, oncard=None, end_session=True): - if oncard == 'carda' and not self._card_a_prefix: - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - elif oncard == 'cardb' and not self._card_b_prefix: - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - elif oncard and oncard != 'carda' and oncard != 'cardb': - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - - db = self.__class__.CACHE_XML if oncard else self.__class__.MEDIA_XML - prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix - bl = BookList(open(prefix + db, 'rb'), prefix, self.report_progress) - paths = bl.purge_corrupted_files() - for path in paths: - path = os.path.join(prefix, path) - if os.path.exists(path): - os.unlink(path) - self.report_progress(1.0, _('Getting list of books on device...')) - return bl - def filename_callback(self, fname, mi): if getattr(mi, 'application_id', None) is not None: base = fname.rpartition('.')[0] @@ -129,90 +76,17 @@ class PRS505(CLI, Device): fname = base + suffix + '.' + fname.rpartition('.')[-1] return fname - def upload_books(self, files, names, on_card=None, end_session=True, - metadata=None): - - path = self._sanity_check(on_card, files) - - paths, ctimes, sizes = [], [], [] - names = iter(names) - metadata = iter(metadata) - for i, infile in enumerate(files): - mdata, fname = metadata.next(), names.next() - filepath = self.create_upload_path(path, mdata, fname) - - paths.append(filepath) - self.put_file(infile, paths[-1], replace_file=True) - ctimes.append(os.path.getctime(paths[-1])) - sizes.append(os.stat(paths[-1]).st_size) - - self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) - - self.report_progress(1.0, _('Transferring books to device...')) - - return zip(paths, sizes, ctimes, cycle([on_card])) - - def add_books_to_metadata(self, locations, metadata, booklists): - if not locations or not metadata: - return - - metadata = iter(metadata) - for location in locations: - info = metadata.next() - path = location[0] - oncard = location[3] - blist = 2 if oncard == 'cardb' else 1 if oncard == 'carda' else 0 - - if self._main_prefix and path.startswith(self._main_prefix): - name = path.replace(self._main_prefix, '') - elif self._card_a_prefix and path.startswith(self._card_a_prefix): - name = path.replace(self._card_a_prefix, '') - elif self._card_b_prefix and path.startswith(self._card_b_prefix): - name = path.replace(self._card_b_prefix, '') - - name = name.replace('\\', '/') - name = name.replace('//', '/') - if name.startswith('/'): - name = name[1:] - - opts = self.settings() - collections = opts.extra_customization.split(',') if opts.extra_customization else [] - booklist = booklists[blist] - if not hasattr(booklist, 'add_book'): - raise ValueError(('Incorrect upload location %s. Did you choose the' - ' correct card A or B, to send books to?')%oncard) - booklist.add_book(info, name, collections, *location[1:-1]) - fix_ids(*booklists) - - def delete_books(self, paths, end_session=True): - for i, path in enumerate(paths): - self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) - if os.path.exists(path): - os.unlink(path) - try: - os.removedirs(os.path.dirname(path)) - except: - pass - self.report_progress(1.0, _('Removing books from device...')) - - @classmethod - def remove_books_from_metadata(cls, paths, booklists): - for path in paths: - for bl in booklists: - if hasattr(bl, 'remove_book'): - bl.remove_book(path) - fix_ids(*booklists) - def sync_booklists(self, booklists, end_session=True): + print 'in sync_booklists' fix_ids(*booklists) if not os.path.exists(self._main_prefix): os.makedirs(self._main_prefix) - with open(self._main_prefix + self.__class__.MEDIA_XML, 'wb') as f: + with open(self._main_prefix + MEDIA_XML, 'wb') as f: booklists[0].write(f) def write_card_prefix(prefix, listid): if prefix is not None and hasattr(booklists[listid], 'write'): - tgt = os.path.join(prefix, *(self.CACHE_XML.split('/'))) + tgt = os.path.join(prefix, *(CACHE_XML.split('/'))) base = os.path.dirname(tgt) if not os.path.exists(base): os.makedirs(base) @@ -221,8 +95,7 @@ class PRS505(CLI, Device): write_card_prefix(self._card_a_prefix, 1) write_card_prefix(self._card_b_prefix, 2) - self.report_progress(1.0, _('Sending metadata to device...')) - + USBMS.sync_booklists(self, booklists, end_session) class PRS700(PRS505): @@ -241,5 +114,3 @@ class PRS700(PRS505): OSX_MAIN_MEM = re.compile(r'Sony PRS-((700/[^:]+)|((6|9)00)) Media') OSX_CARD_A_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))MS Media') OSX_CARD_B_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))SD Media') - - diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 990b335a6d..bc6003de27 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -33,7 +33,9 @@ class Book(MetaInformation): self.path = os.path.join(prefix, lpath) if os.sep == '\\': self.path = self.path.replace('/', '\\') - self.lpath = lpath + self.lpath = lpath.replace('\\', '/') + else: + self.lpath = lpath self.mime = mime_type_ext(path_to_ext(lpath)) self.size = os.stat(self.path).st_size if size == None else size self.db_id = None diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 63d28f5457..1a5b7461ed 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -31,32 +31,36 @@ class USBMS(CLI, Device): CAN_SET_METADATA = True METADATA_CACHE = 'metadata.calibre' + def initialize(self): + Device.initialize(self) + self.booklist_class = BookList + def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) return (self.get_gui_name(), '', '', '') def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext - bl = BookList() - metadata = BookList() - need_sync = False if oncard == 'carda' and not self._card_a_prefix: self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] elif oncard == 'cardb' and not self._card_b_prefix: self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] elif oncard and oncard != 'carda' and oncard != 'cardb': self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix + metadata = self.booklist_class(oncard, prefix) + ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ self.get_main_ebook_dir() - bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE) + bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE, + self.booklist_class(oncard, prefix)) # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} @@ -109,7 +113,6 @@ class USBMS(CLI, Device): # find on the device. If need_sync is True then there were either items # on the device that were not in bl or some of the items were changed. if self.count_found_in_bl != len(bl) or need_sync: - print 'resync' if oncard == 'cardb': self.sync_booklists((None, None, metadata)) elif oncard == 'carda': @@ -122,7 +125,6 @@ class USBMS(CLI, Device): def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): - path = self._sanity_check(on_card, files) paths = [] @@ -145,7 +147,6 @@ class USBMS(CLI, Device): self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) self.report_progress(1.0, _('Transferring books to device...')) - return zip(paths, cycle([on_card])) def upload_cover(self, path, filename, metadata): @@ -174,11 +175,10 @@ class USBMS(CLI, Device): lpath = path.partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - lpath = lpath.replace('\\', '/') book = Book(prefix, lpath, other=info) - - if book not in booklists[blist]: - booklists[blist].append(book) + opts = self.settings() + collections = opts.extra_customization.split(',') if opts.extra_customization else [] + booklists[blist].add_book(book, collections, *location[1:-1]) self.report_progress(1.0, _('Adding books to device metadata listing...')) @@ -209,7 +209,7 @@ class USBMS(CLI, Device): for bl in booklists: for book in bl: if path.endswith(book.path): - bl.remove(book) + bl.remove_book(book) self.report_progress(1.0, _('Removing books from device metadata listing...')) def sync_booklists(self, booklists, end_session=True): @@ -217,7 +217,7 @@ class USBMS(CLI, Device): os.makedirs(self._main_prefix) def write_prefix(prefix, listid): - if prefix is not None and isinstance(booklists[listid], BookList): + if prefix is not None and isinstance(booklists[listid], self.booklist_class): if not os.path.exists(prefix): os.makedirs(prefix) js = [item.to_json() for item in booklists[listid]] @@ -230,9 +230,8 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) @classmethod - def parse_metadata_cache(cls, prefix, name): + def parse_metadata_cache(cls, prefix, name, bl): js = [] - bl = BookList() need_sync = False try: with open(os.path.join(prefix, name), 'rb') as f: @@ -249,7 +248,7 @@ class USBMS(CLI, Device): except: import traceback traceback.print_exc() - bl = BookList() + bl = [] return bl, need_sync @classmethod diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 60dffc0cf7..a1c29be337 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -254,11 +254,11 @@ class MetaInformation(object): setattr(self, x, getattr(mi, x, None)) def print_all_attributes(self): - for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', - 'series', 'series_index', 'rating', 'isbn', 'language', + for x in ('author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher', + 'series', 'series_index', 'tags', 'rating', 'isbn', 'language', 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', - 'rights', 'publication_type', 'uuid', + 'rights', 'publication_type', 'uuid', 'tag_order', ): prints(x, getattr(self, x, 'None')) @@ -278,7 +278,7 @@ class MetaInformation(object): 'isbn', 'application_id', 'manifest', 'spine', 'toc', 'cover', 'language', 'guide', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights', - 'publication_type', 'uuid',): + 'publication_type', 'uuid', 'tag_order'): if hasattr(mi, attr): val = getattr(mi, attr) if val is not None: diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index df355f0ef5..828756e2c8 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -821,7 +821,9 @@ class DeviceGUI(object): def sync_to_device(self, on_card, delete_from_library, specific_format=None, send_ids=None, do_auto_convert=True): - ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids + ids = [self.library_view.model().id(r) \ + for r in self.library_view.selectionModel().selectedRows()] \ + if send_ids is None else send_ids if not self.device_manager or not ids or len(ids) == 0: return @@ -842,8 +844,7 @@ class DeviceGUI(object): ids = iter(ids) for mi in metadata: if mi.cover and os.access(mi.cover, os.R_OK): - mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, - 'rb').read()) + mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read()) imetadata = iter(metadata) files = [getattr(f, 'name', None) for f in _files] @@ -890,7 +891,9 @@ class DeviceGUI(object): bad.append(self.library_view.model().db.title(id, index_is_id=True)) if auto != []: - format = specific_format if specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))) else None + format = specific_format if specific_format in \ + list(set(settings.format_map).intersection(set(available_output_formats()))) \ + else None if not format: for fmt in settings.format_map: if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))): @@ -1039,7 +1042,8 @@ class DeviceGUI(object): loc[i] = True break if mi.authors and \ - re.sub('(?u)\W|[_]', '', mi.authors.lower()) in cache['authors']: + re.sub('(?u)\W|[_]', '', authors_to_string(mi.authors).lower()) \ + in cache['authors']: loc[i] = True break return loc diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index cdebf65489..e116e39397 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -17,7 +17,7 @@ from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, pyqtSignal, \ SIGNAL, QObject, QSize, QModelIndex, QDate from calibre import strftime -from calibre.ebooks.metadata import fmt_sidx, authors_to_string +from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.gui2 import NONE, TableView, config, error_dialog, UNDEFINED_QDATE from calibre.gui2.dialogs.comments_dialog import CommentsDialog @@ -1378,7 +1378,10 @@ class DeviceBooksModel(BooksModel): def libcmp(x, y): x, y = self.db[x].in_library, self.db[y].in_library return cmp(x, y) - fcmp = strcmp('title_sorter') if col == 0 else strcmp('authors') if col == 1 else \ + def authorcmp(x, y): + x, y = authors_to_string(self.db[x].authors), authors_to_string(self.db[y].authors) + return cmp(x, y) + fcmp = strcmp('title_sorter') if col == 0 else authorcmp if col == 1 else \ sizecmp if col == 2 else datecmp if col == 3 else tagscmp if col == 4 else libcmp self.map.sort(cmp=fcmp, reverse=descending) if len(self.map) == len(self.db): @@ -1446,9 +1449,9 @@ class DeviceBooksModel(BooksModel): au = self.db[self.map[row]].authors if not au: au = self.unknown - if role == Qt.EditRole: - return QVariant(authors_to_string(au)) - return QVariant(" & ".join(au)) +# if role == Qt.EditRole: +# return QVariant(au) + return QVariant(authors_to_string(au)) elif col == 2: size = self.db[self.map[row]].size return QVariant(BooksView.human_readable(size)) @@ -1501,7 +1504,7 @@ class DeviceBooksModel(BooksModel): self.db[idx].title = val self.db[idx].title_sorter = val elif col == 1: - self.db[idx].authors = val + self.db[idx].authors = string_to_authors(val) elif col == 4: tags = [i.strip() for i in val.split(',')] tags = [t for t in tags if t] From 5c9e2ae267c4f178047c45d7b473a51f77001d4c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Thu, 13 May 2010 22:17:52 +0100 Subject: [PATCH 04/20] After pylint --- src/calibre/devices/prs505/books.py | 7 ++++++- src/calibre/devices/prs505/driver.py | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 82bc977bcd..ba3605530e 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -110,7 +110,7 @@ class BookList(_BookList): return child return None - def add_book(self, book, collections): + def add_book(self, book, collections=None): if book in self: return """ Add a node into the DOM tree, representing a book """ @@ -267,6 +267,11 @@ class BookList(_BookList): pli.parentNode.removeChild(pli) pli.unlink() + def set_tags(self, book, tags): + tags = [t for t in tags if t] + book.tags = tags + self.set_playlists(book.id, tags) + def set_playlists(self, id, collections): self.remove_from_playlists(id) for collection in set(collections): diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 3e1ee67faa..0b41894a18 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -77,7 +77,6 @@ class PRS505(USBMS): return fname def sync_booklists(self, booklists, end_session=True): - print 'in sync_booklists' fix_ids(*booklists) if not os.path.exists(self._main_prefix): os.makedirs(self._main_prefix) From 72fbd67c1764f1c1fc9e7dad00ffa144fab2bf04 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 14 May 2010 12:17:14 +0100 Subject: [PATCH 05/20] More testing of: 1) initial condition: cache does not exist on book 2) adding and removing books 3) subsequent conditions: cache exists In addition: 1) added metadata correction for books matched with something other than UUID. 2) Refactored changes to BookList, to move the additional methods into USMBS from Interface. 3) Made classmethods in USBMS into normal methods. --- src/calibre/devices/interface.py | 14 -------- src/calibre/devices/prs505/books.py | 39 ++++------------------ src/calibre/devices/prs505/driver.py | 4 ++- src/calibre/devices/usbms/books.py | 39 ++++++++++++++-------- src/calibre/devices/usbms/driver.py | 50 +++++++++++++--------------- src/calibre/gui2/device.py | 25 ++++++++++---- src/calibre/library/database2.py | 8 ++--- 7 files changed, 82 insertions(+), 97 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 6247e29e15..b38b62e20c 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -402,17 +402,3 @@ class BookList(list): ''' raise NotImplementedError() - def add_book(self, book, collections=None): - ''' - Add the book to the booklist. Intent is to maintain any device-internal - metadata - ''' - if book not in self: - self.append(book) - - def remove_book(self, book): - ''' - Remove a book from the booklist. Correct any device metadata at the - same time - ''' - self.remove(book) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index ba3605530e..855d8d5cd3 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -8,7 +8,7 @@ import xml.dom.minidom as dom from base64 import b64encode as encode -from calibre.devices.interface import BookList as _BookList +from calibre.devices.usbms.books import BookList as _BookList from calibre.devices import strftime as _strftime from calibre.devices.usbms.books import Book as _Book from calibre.devices.prs505 import MEDIA_XML @@ -31,36 +31,6 @@ def uuid(): def sortable_title(title): return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip() -class book_metadata_field(object): - """ Represents metadata stored as an attribute """ - def __init__(self, attr, formatter=None, setter=None): - self.attr = attr - self.formatter = formatter - self.setter = setter - - def __get__(self, obj, typ=None): - """ Return a string. String may be empty if self.attr is absent """ - return self.formatter(obj.elem.getAttribute(self.attr)) if \ - self.formatter else obj.elem.getAttribute(self.attr).strip() - - def __set__(self, obj, val): - """ Set the attribute """ - val = self.setter(val) if self.setter else val - if not isinstance(val, unicode): - val = unicode(val, 'utf8', 'replace') - obj.elem.setAttribute(self.attr, val) - - -class Book(_Book): - @dynamic_property - def db_id(self): - doc = '''The database id in the application database that this file corresponds to''' - def fget(self): - match = re.search(r'_(\d+)$', self.rpath.rpartition('.')[0]) - if match: - return int(match.group(1)) - return property(fget=fget, doc=doc) - class BookList(_BookList): def __init__(self, oncard, prefix): @@ -318,7 +288,12 @@ class BookList(_BookList): imap = {} for book, sony_id in zip(books, sony_ids): if book is not None: - imap[book.application_id] = sony_id + db_id = book.application_id + if db_id is None: + db_id = book.db_id + print 'here', db_id + if db_id is not None: + imap[book.application_id] = sony_id # filter the list, removing books not on device but on playlist books = [i for i in books if i is not None] # filter the order specification to the books we have diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 0b41894a18..1d403cb75d 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -13,6 +13,7 @@ import os import re from calibre.devices.usbms.driver import USBMS +from calibre.devices.usbms.books import Book from calibre.devices.prs505.books import BookList, fix_ids from calibre.devices.prs505 import MEDIA_XML from calibre.devices.prs505 import CACHE_XML @@ -59,8 +60,9 @@ class PRS505(USBMS): METADATA_CACHE = "database/cache/metadata.calibre" def initialize(self): - USBMS.initialize(self) + USBMS.initialize(self) # Must be first, so _class vars are set right self.booklist_class = BookList + self.book_class = Book def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index bc6003de27..ce74db6f54 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -37,12 +37,8 @@ class Book(MetaInformation): else: self.lpath = lpath self.mime = mime_type_ext(path_to_ext(lpath)) - self.size = os.stat(self.path).st_size if size == None else size - self.db_id = None - try: - self.datetime = time.gmtime(os.path.getctime(self.path)) - except ValueError: - self.datetime = time.gmtime() + self.size = None # will be set later + self.datetime = time.gmtime() if other: self.smart_update(other) @@ -70,6 +66,16 @@ class Book(MetaInformation): return spath == opath + @dynamic_property + def db_id(self): + doc = '''The database id in the application database that this file corresponds to''' + def fget(self): + match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0]) + if match: + return int(match.group(1)) + return None + return property(fget=fget, doc=doc) + @dynamic_property def title_sorter(self): doc = '''String to sort the title. If absent, title is returned''' @@ -81,13 +87,6 @@ class Book(MetaInformation): def thumbnail(self): return None -# def __str__(self): -# ''' -# Return a utf-8 encoded string with title author and path information -# ''' -# return self.title.encode('utf-8') + " by " + \ -# self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') - def smart_update(self, other): ''' Merge the information in C{other} into self. In case of conflicts, the information @@ -115,3 +114,17 @@ class BookList(_BookList): def set_tags(self, book, tags): book.tags = tags + def add_book(self, book, collections=None): + ''' + Add the book to the booklist. Intent is to maintain any device-internal + metadata + ''' + if book not in self: + self.append(book) + + def remove_book(self, book): + ''' + Remove a book from the booklist. Correct any device metadata at the + same time + ''' + self.remove(book) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 1a5b7461ed..361f7ea1bf 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -34,6 +34,7 @@ class USBMS(CLI, Device): def initialize(self): Device.initialize(self) self.booklist_class = BookList + self.book_class = Book def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) @@ -52,7 +53,9 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Getting list of books on device...')) return [] - prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix + prefix = self._card_a_prefix if oncard == 'carda' else \ + self._card_b_prefix if oncard == 'cardb' \ + else self._main_prefix metadata = self.booklist_class(oncard, prefix) ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ @@ -61,7 +64,6 @@ class USBMS(CLI, Device): bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE, self.booklist_class(oncard, prefix)) - # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): @@ -77,10 +79,10 @@ class USBMS(CLI, Device): lpath = lpath[len(os.sep):] idx = bl_cache.get(lpath.replace('\\', '/'), None) if idx is not None: - item, changed = self.__class__.update_metadata_item(bl[idx]) + item, changed = self.update_metadata_item(bl[idx]) self.count_found_in_bl += 1 else: - item = self.__class__.book_from_path(prefix, lpath) + item = self.book_from_path(prefix, lpath) changed = True metadata.append(item) except: # Probably a filename encoding error @@ -175,7 +177,10 @@ class USBMS(CLI, Device): lpath = path.partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - book = Book(prefix, lpath, other=info) + book = self.book_class(prefix, lpath, other=info) + if book.size is None: + book.size = os.stat(path).st_size + opts = self.settings() collections = opts.extra_customization.split(',') if opts.extra_customization else [] booklists[blist].add_book(book, collections, *location[1:-1]) @@ -229,19 +234,14 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) - @classmethod - def parse_metadata_cache(cls, prefix, name, bl): + def parse_metadata_cache(self, prefix, name, bl): js = [] need_sync = False try: with open(os.path.join(prefix, name), 'rb') as f: js = json.load(f, encoding='utf-8') for item in js: - lpath = item.get('lpath', None) - if not lpath or not os.path.exists(os.path.join(prefix, lpath)): - need_sync = True - continue - book = Book(prefix, lpath) + book = self.book_class(prefix, item.get('lpath', None)) for key in item.keys(): setattr(book, key, item[key]) bl.append(book) @@ -249,35 +249,33 @@ class USBMS(CLI, Device): import traceback traceback.print_exc() bl = [] + need_sync = True return bl, need_sync - @classmethod - def update_metadata_item(cls, item): + def update_metadata_item(self, item): changed = False size = os.stat(item.path).st_size if size != item.size: changed = True - mi = cls.metadata_from_path(item.path) + mi = self.metadata_from_path(item.path) item.smart_update(mi) + item.size = size return item, changed - @classmethod - def metadata_from_path(cls, path): - return cls.metadata_from_formats([path]) + def metadata_from_path(self, path): + return self.metadata_from_formats([path]) - @classmethod - def metadata_from_formats(cls, fmts): + def metadata_from_formats(self, fmts): from calibre.ebooks.metadata.meta import metadata_from_formats from calibre.customize.ui import quick_metadata with quick_metadata: return metadata_from_formats(fmts) - @classmethod - def book_from_path(cls, prefix, path): + def book_from_path(self, prefix, path): from calibre.ebooks.metadata import MetaInformation - if cls.settings().read_metadata or cls.MUST_READ_METADATA: - mi = cls.metadata_from_path(os.path.join(prefix, path)) + if self.settings().read_metadata or self.MUST_READ_METADATA: + mi = self.metadata_from_path(os.path.join(prefix, path)) else: from calibre.ebooks.metadata.meta import metadata_from_filename mi = metadata_from_filename(os.path.basename(path), @@ -286,6 +284,6 @@ class USBMS(CLI, Device): if mi is None: mi = MetaInformation(os.path.splitext(os.path.basename(path))[0], [_('Unknown')]) - - book = Book(prefix, path, other=mi) + mi.size = os.stat(os.path.join(prefix, path)).st_size + book = self.book_class(prefix, path, other=mi) return book diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 828756e2c8..a9f69bbca5 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -523,7 +523,8 @@ class DeviceGUI(object): d = ChooseFormatDialog(self, _('Choose format to send to device'), self.device_manager.device.settings().format_map) d.exec_() - fmt = d.format().lower() + if d.format(): + fmt = d.format().lower() dest, sub_dest = dest.split(':') if dest in ('main', 'carda', 'cardb'): if not self.device_connected or not self.device_manager: @@ -998,7 +999,7 @@ class DeviceGUI(object): if changed: self.library_view.model().refresh_ids(list(changed)) - def book_on_device(self, index, format=None, reset=False): + def book_on_device(self, id, format=None, reset=False): loc = [None, None, None] if reset: @@ -1030,7 +1031,7 @@ class DeviceGUI(object): if uuid is not None: self.book_db_uuid_cache[i].add(uuid) - mi = self.library_view.model().db.get_metadata(index, index_is_id=True) + mi = self.library_view.model().db.get_metadata(id, index_is_id=True) for i, l in enumerate(self.booklists()): if mi.uuid in self.book_db_uuid_cache[i]: loc[i] = True @@ -1038,7 +1039,7 @@ class DeviceGUI(object): db_title = re.sub('(?u)\W|[_]', '', mi.title.lower()) cache = self.book_db_title_cache[i].get(db_title, None) if cache: - if index in cache['db_ids']: + if id in cache['db_ids']: loc[i] = True break if mi.authors and \ @@ -1057,11 +1058,11 @@ class DeviceGUI(object): mi = self.library_view.model().db.get_metadata(idx, index_is_id=False) title = re.sub('(?u)\W|[_]', '', mi.title.lower()) if title not in self.db_book_title_cache: - self.db_book_title_cache[title] = {'authors':set(), 'db_ids':set()} + self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}} authors = authors_to_string(mi.authors).lower() if mi.authors else '' authors = re.sub('(?u)\W|[_]', '', authors) - self.db_book_title_cache[title]['authors'].add(authors) - self.db_book_title_cache[title]['db_ids'].add(mi.application_id) + self.db_book_title_cache[title]['authors'][authors] = mi + self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi self.db_book_uuid_cache.add(mi.uuid) # Now iterate through all the books on the device, setting the in_library field @@ -1069,6 +1070,7 @@ class DeviceGUI(object): # is really the db key, but as this can accidentally match across libraries we # also verify the title. The db_id exists on Sony devices. Fallback is title # and author match + resend_metadata = False for booklist in booklists: for book in booklist: if getattr(book, 'uuid', None) in self.db_book_uuid_cache: @@ -1082,11 +1084,20 @@ class DeviceGUI(object): if d is not None: if getattr(book, 'application_id', None) in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.application_id]) + resend_metadata = True continue if book.db_id in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.db_id]) + resend_metadata = True continue book_authors = authors_to_string(book.authors).lower() if book.authors else '' book_authors = re.sub('(?u)\W|[_]', '', book_authors) if book_authors in d['authors']: book.in_library = True + book.smart_update(d['authors'][book_authors]) + resend_metadata = True + if resend_metadata: + # Correcting metadata cache on device. + self.device_manager.sync_booklists(None, booklists) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index fd59503eed..b0f2d3cb39 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -470,14 +470,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): im = PILImage.open(f) im.convert('RGB').save(path, 'JPEG') - def book_on_device(self, index): + def book_on_device(self, id): if callable(self.book_on_device_func): - return self.book_on_device_func(index) + return self.book_on_device_func(id) return None - def book_on_device_string(self, index): + def book_on_device_string(self, id): loc = [] - on = self.book_on_device(index) + on = self.book_on_device(id) if on is not None: m, a, b = on if m is not None: From 96079a712ccaf9369f5aa766c432f6f2adaefad9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 14 May 2010 14:28:53 +0100 Subject: [PATCH 06/20] Put @classmethods back in. I don't understand why the decoration is there, so I hesitate to take it out. Far as I can see, there is no reason for it, but ... --- src/calibre/devices/usbms/driver.py | 34 ++++++++++++++++++----------- src/calibre/gui2/device.py | 10 ++++----- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 361f7ea1bf..1fdf3bdf84 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -27,14 +27,17 @@ class USBMS(CLI, Device): author = _('John Schember') supported_platforms = ['windows', 'osx', 'linux'] + booklist_class = BookList + book_class = Book + FORMATS = [] CAN_SET_METADATA = True METADATA_CACHE = 'metadata.calibre' def initialize(self): Device.initialize(self) - self.booklist_class = BookList - self.book_class = Book +# self.booklist_class = BookList +# self.book_class = Book def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) @@ -234,14 +237,15 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) - def parse_metadata_cache(self, prefix, name, bl): + @classmethod + def parse_metadata_cache(cls, prefix, name, bl): js = [] need_sync = False try: with open(os.path.join(prefix, name), 'rb') as f: js = json.load(f, encoding='utf-8') for item in js: - book = self.book_class(prefix, item.get('lpath', None)) + book = cls.book_class(prefix, item.get('lpath', None)) for key in item.keys(): setattr(book, key, item[key]) bl.append(book) @@ -252,30 +256,34 @@ class USBMS(CLI, Device): need_sync = True return bl, need_sync - def update_metadata_item(self, item): + @classmethod + def update_metadata_item(cls, item): changed = False size = os.stat(item.path).st_size if size != item.size: changed = True - mi = self.metadata_from_path(item.path) + mi = cls.metadata_from_path(item.path) item.smart_update(mi) item.size = size return item, changed - def metadata_from_path(self, path): - return self.metadata_from_formats([path]) + @classmethod + def metadata_from_path(cls, path): + return cls.metadata_from_formats([path]) - def metadata_from_formats(self, fmts): + @classmethod + def metadata_from_formats(cls, fmts): from calibre.ebooks.metadata.meta import metadata_from_formats from calibre.customize.ui import quick_metadata with quick_metadata: return metadata_from_formats(fmts) - def book_from_path(self, prefix, path): + @classmethod + def book_from_path(cls, prefix, path): from calibre.ebooks.metadata import MetaInformation - if self.settings().read_metadata or self.MUST_READ_METADATA: - mi = self.metadata_from_path(os.path.join(prefix, path)) + if cls.settings().read_metadata or cls.MUST_READ_METADATA: + mi = cls.metadata_from_path(os.path.join(prefix, path)) else: from calibre.ebooks.metadata.meta import metadata_from_filename mi = metadata_from_filename(os.path.basename(path), @@ -285,5 +293,5 @@ class USBMS(CLI, Device): mi = MetaInformation(os.path.splitext(os.path.basename(path))[0], [_('Unknown')]) mi.size = os.stat(os.path.join(prefix, path)).st_size - book = self.book_class(prefix, path, other=mi) + book = cls.book_class(prefix, path, other=mi) return book diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a9f69bbca5..af314c5468 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1022,11 +1022,11 @@ class DeviceGUI(object): book_authors = authors_to_string(book.authors).lower() book_authors = re.sub('(?u)\W|[_]', '', book_authors) self.book_db_title_cache[i][book_title]['authors'].add(book_authors) - id = getattr(book, 'application_id', None) - if id is None: - id = book.db_id - if id is not None: - self.book_db_title_cache[i][book_title]['db_ids'].add(id) + db_id = getattr(book, 'application_id', None) + if db_id is None: + db_id = book.db_id + if db_id is not None: + self.book_db_title_cache[i][book_title]['db_ids'].add(db_id) uuid = getattr(book, 'uuid', None) if uuid is not None: self.book_db_uuid_cache[i].add(uuid) From dc130d56a92f0250f1e49df53eaa9a7054afd75f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 14 May 2010 14:52:33 +0100 Subject: [PATCH 07/20] Cleanup, remove some print statements --- src/calibre/devices/prs505/books.py | 1 - src/calibre/devices/prs505/driver.py | 10 +++------- src/calibre/devices/usbms/driver.py | 9 ++++----- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 855d8d5cd3..7f4071a6cf 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -291,7 +291,6 @@ class BookList(_BookList): db_id = book.application_id if db_id is None: db_id = book.db_id - print 'here', db_id if db_id is not None: imap[book.application_id] = sony_id # filter the list, removing books not on device but on playlist diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 1d403cb75d..9ff88da592 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -13,8 +13,7 @@ import os import re from calibre.devices.usbms.driver import USBMS -from calibre.devices.usbms.books import Book -from calibre.devices.prs505.books import BookList, fix_ids +from calibre.devices.prs505.books import BookList as PRS_BookList, fix_ids from calibre.devices.prs505 import MEDIA_XML from calibre.devices.prs505 import CACHE_XML from calibre import __appname__ @@ -28,6 +27,8 @@ class PRS505(USBMS): supported_platforms = ['windows', 'osx', 'linux'] path_sep = '/' + booklist_class = PRS_BookList # See USBMS for some explanation of this + FORMATS = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt'] VENDOR_ID = [0x054c] #: SONY Vendor Id @@ -59,11 +60,6 @@ class PRS505(USBMS): METADATA_CACHE = "database/cache/metadata.calibre" - def initialize(self): - USBMS.initialize(self) # Must be first, so _class vars are set right - self.booklist_class = BookList - self.book_class = Book - def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 1fdf3bdf84..64b27a993d 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -27,6 +27,10 @@ class USBMS(CLI, Device): author = _('John Schember') supported_platforms = ['windows', 'osx', 'linux'] + # Store type instances of BookList and Book. We must do this because + # a) we need to override these classes in some device drivers, and + # b) the classmethods seem only to see real attributes declared in the + # class, not attributes stored in the class booklist_class = BookList book_class = Book @@ -34,11 +38,6 @@ class USBMS(CLI, Device): CAN_SET_METADATA = True METADATA_CACHE = 'metadata.calibre' - def initialize(self): - Device.initialize(self) -# self.booklist_class = BookList -# self.book_class = Book - def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) return (self.get_gui_name(), '', '', '') From 9a0dfff78e93857ed4bf4dad44efff506ba6f913 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 13:37:39 +0100 Subject: [PATCH 08/20] 1) improve performance of OnDevice refresh. Instead of rebuilding the complete book list, iterate through the existing one and set the value of OnDevice correctly. 2) Fix problems with Sony readers and metadata caching. Needed to ensure that when a book is added to the booklist from the JSON cache, it is added to the sony cache if it isn't already there. 3) Build the sony metadata maps (caches) on the fly instead of in reorder_playlists. 4) Refactor method declarations. 5) Move the JSON cache to the root of the card for Sony devices. --- src/calibre/devices/prs505/books.py | 78 ++++++++++++++++------------ src/calibre/devices/prs505/driver.py | 2 - src/calibre/devices/usbms/books.py | 7 ++- src/calibre/devices/usbms/driver.py | 19 +++---- src/calibre/gui2/library.py | 2 +- src/calibre/gui2/ui.py | 2 +- src/calibre/library/caches.py | 6 +++ src/calibre/library/database2.py | 2 + 8 files changed, 71 insertions(+), 47 deletions(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 7f4071a6cf..40a98913be 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -10,9 +10,8 @@ from base64 import b64encode as encode from calibre.devices.usbms.books import BookList as _BookList from calibre.devices import strftime as _strftime -from calibre.devices.usbms.books import Book as _Book -from calibre.devices.prs505 import MEDIA_XML -from calibre.devices.prs505 import CACHE_XML +from calibre.devices.prs505 import MEDIA_XML, CACHE_XML +from calibre.devices.errors import PathError strftime = functools.partial(_strftime, zone=time.gmtime) @@ -33,10 +32,14 @@ def sortable_title(title): class BookList(_BookList): - def __init__(self, oncard, prefix): - _BookList.__init__(self, oncard, prefix) + def __init__(self, oncard, prefix, settings): + _BookList.__init__(self, oncard, prefix, settings) if prefix is None: return + self.sony_id_cache = {} + self.books_lpath_cache = {} + opts = settings() + self.collections = opts.extra_customization.split(',') if opts.extra_customization else [] db = CACHE_XML if oncard else MEDIA_XML xml_file = open(prefix + db, 'rb') xml_file.seek(0) @@ -50,8 +53,21 @@ class BookList(_BookList): self.root_element = records[0] else: self.prefix = '' + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + self.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + # set the key to none. Will be filled in later when booklist is built + self.books_lpath_cache[child.getAttribute('path')] = None self.tag_order = {} + paths = self.purge_corrupted_files() + for path in paths: + try: + self.del_file(path, end_session=False) + except PathError: # Incase this is a refetch without a sync in between + continue + + def max_id(self): max = 0 for child in self.root_element.childNodes: @@ -73,22 +89,27 @@ class BookList(_BookList): def supports_tags(self): return True - def book_by_path(self, path): - for child in self.root_element.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"): - if path == child.getAttribute('path'): - return child - return None - - def add_book(self, book, collections=None): + def add_book(self, book, replace_metadata): + # Add a node into the DOM tree, representing a book. Also add to booklist if book in self: - return - """ Add a node into the DOM tree, representing a book """ - node = self.document.createElement(self.prefix + "text") - mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) + # replacing metadata for book + self.delete_node(book.lpath) + else: + self.append(book) + if not replace_metadata: + if self.books_lpath_cache.has_key(book.lpath): + self.books_lpath_cache[book.lpath] = book + return + # Book not in metadata. Add it. Note that we don't need to worry about + # extra books in the Sony metadata. The reader deletes them for us when + # we disconnect. That said, if it becomes important one day, we can do + # it by scanning the books_lpath_cache for None entries and removing the + # corresponding nodes. + self.books_lpath_cache[book.lpath] = book cid = self.max_id()+1 - book.sony_id = cid - self.append(book) + node = self.document.createElement(self.prefix + "text") + self.sony_id_cache[cid] = book.lpath + mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) try: sourceid = str(self[0].sourceid) if len(self) else '1' except: @@ -120,7 +141,7 @@ class BookList(_BookList): self.root_element.appendChild(node) tags = [] - for item in collections: + for item in self.collections: item = item.strip() mitem = getattr(book, item, None) titems = [] @@ -141,6 +162,7 @@ class BookList(_BookList): if hasattr(book, 'tag_order'): self.tag_order.update(book.tag_order) self.set_playlists(cid, tags) + return True # metadata cache has changed. Must sync at end def _delete_node(self, node): nid = node.getAttribute('id') @@ -162,7 +184,8 @@ class BookList(_BookList): def remove_book(self, book): ''' Remove DOM node corresponding to book with C{path == path}. - Also remove book from any collections it is part of. + Also remove book from any collections it is part of, and remove + from the booklist ''' self.remove(book) self.delete_node(book.lpath) @@ -264,15 +287,6 @@ class BookList(_BookList): stream.write(src.replace("'", ''')) def reorder_playlists(self): - sony_id_cache = {} - for child in self.root_element.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): - sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') - - books_lpath_cache = {} - for book in self: - books_lpath_cache[book.lpath] = book - for title in self.tag_order.keys(): pl = self.playlist_by_title(title) if not pl: @@ -281,9 +295,9 @@ class BookList(_BookList): sony_ids = [id.getAttribute('id') \ for id in pl.childNodes if hasattr(id, 'getAttribute')] # convert IDs in playlist to a list of lpaths - sony_paths = [sony_id_cache[id] for id in sony_ids] + sony_paths = [self.sony_id_cache[id] for id in sony_ids] # create list of books containing lpaths - books = [books_lpath_cache.get(p, None) for p in sony_paths] + books = [self.books_lpath_cache.get(p, None) for p in sony_paths] # create dict of db_id -> sony_id imap = {} for book, sony_id in zip(books, sony_ids): diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 9ff88da592..d2823ff4a4 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -58,8 +58,6 @@ class PRS505(USBMS): 'series, tags, authors' EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags']) - METADATA_CACHE = "database/cache/metadata.calibre" - def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index ce74db6f54..b153300282 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -108,16 +108,19 @@ class Book(MetaInformation): class BookList(_BookList): + def __init__(self, oncard, prefix, settings): + pass + def supports_tags(self): return True def set_tags(self, book, tags): book.tags = tags - def add_book(self, book, collections=None): + def add_book(self, book, replace_metadata): ''' Add the book to the booklist. Intent is to maintain any device-internal - metadata + metadata. Return True if booklists must be sync'ed ''' if book not in self: self.append(book) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 64b27a993d..95b7441f44 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -58,20 +58,22 @@ class USBMS(CLI, Device): prefix = self._card_a_prefix if oncard == 'carda' else \ self._card_b_prefix if oncard == 'cardb' \ else self._main_prefix - metadata = self.booklist_class(oncard, prefix) ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ self.get_main_ebook_dir() - bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE, - self.booklist_class(oncard, prefix)) + # build a temporary list of books from the metadata cache + bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE) # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): bl_cache[b.lpath] = idx self.count_found_in_bl = 0 + # Make the real booklist that will be filled in below + metadata = self.booklist_class(oncard, prefix, self.settings) + def update_booklist(filename, path, prefix): changed = False if path_to_ext(filename) in self.FORMATS: @@ -86,7 +88,8 @@ class USBMS(CLI, Device): else: item = self.book_from_path(prefix, lpath) changed = True - metadata.append(item) + if metadata.add_book(item, replace_metadata=False): + changed = True except: # Probably a filename encoding error import traceback traceback.print_exc() @@ -183,10 +186,7 @@ class USBMS(CLI, Device): if book.size is None: book.size = os.stat(path).st_size - opts = self.settings() - collections = opts.extra_customization.split(',') if opts.extra_customization else [] - booklists[blist].add_book(book, collections, *location[1:-1]) - + booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) def delete_books(self, paths, end_session=True): @@ -237,7 +237,8 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) @classmethod - def parse_metadata_cache(cls, prefix, name, bl): + def parse_metadata_cache(cls, prefix, name): + bl = [] js = [] need_sync = False try: diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index e116e39397..eeda687312 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -371,7 +371,7 @@ class BooksModel(QAbstractTableModel): def set_device_connected(self, is_connected): self.device_connected = is_connected self.read_config() - self.refresh(reset=True) + self.db.refresh_ondevice() self.database_changed.emit(self.db) def set_book_on_device_func(self, func): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 48e22f8903..ba6bac76e4 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -947,7 +947,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_manager.device) self.location_view.model().device_connected(self.device_manager.device) self.eject_action.setEnabled(True) - self.refresh_ondevice_info (device_connected = True) + # don't refresh_ondevice here. It will happen in metadata_downloaded else: self.save_device_view_settings() self.device_connected = False diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 9ed150733a..acc8eaffb6 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -547,6 +547,12 @@ class ResultCache(SearchQueryParser): def count(self): return len(self._map) + def refresh_ondevice(self, db): + ondevice_col = self.FIELD_MAP['ondevice'] + for item in self._data: + if item is not None: + item[ondevice_col] = db.book_on_device_string(item[0]) + def refresh(self, db, field=None, ascending=True): temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index b0f2d3cb39..5971333078 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) + self.refresh() self.last_update_check = self.last_modified() From 1a42f0aae76b553395eb4d9b1cf9abf00bcb07e2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 20:08:18 +0100 Subject: [PATCH 09/20] First iteration of folder_device. --- src/calibre/devices/folder_device/__init__.py | 10 +++ src/calibre/devices/folder_device/driver.py | 74 +++++++++++++++++++ src/calibre/devices/htc_td2/driver.py | 3 +- src/calibre/gui2/device.py | 35 +++++++++ src/calibre/gui2/ui.py | 14 ++++ 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/calibre/devices/folder_device/__init__.py create mode 100644 src/calibre/devices/folder_device/driver.py diff --git a/src/calibre/devices/folder_device/__init__.py b/src/calibre/devices/folder_device/__init__.py new file mode 100644 index 0000000000..3d1a86922e --- /dev/null +++ b/src/calibre/devices/folder_device/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py new file mode 100644 index 0000000000..700b7f3eec --- /dev/null +++ b/src/calibre/devices/folder_device/driver.py @@ -0,0 +1,74 @@ +''' +Created on 15 May 2010 + +@author: charles +''' +import os +import time + +from calibre.customize.ui import available_output_formats +from calibre.devices.usbms.driver import USBMS, BookList +from calibre.devices.interface import DevicePlugin +from calibre.devices.usbms.deviceconfig import DeviceConfig +from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to + +class FOLDER_DEVICE(USBMS): + type = _('Device Interface') + + # Ordered list of supported formats + FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] + + THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device + # Whether the metadata on books can be set via the GUI. + CAN_SET_METADATA = True + SUPPORTS_SUB_DIRS = True + DELETE_EXTS = [] + #: Path separator for paths to books on device + path_sep = os.sep + #: Icon for this device + icon = I('reader.svg') + METADATA_CACHE = '.metadata.calibre' + + _main_prefix = None + _card_a_prefix = None + _card_b_prefix = None + + def __init__(self, path): + self._main_prefix = path + self.booklist_class = BookList + self.is_connected = True + + @classmethod + def get_gui_name(cls): + if hasattr(cls, 'gui_name'): + return cls.gui_name + if hasattr(cls, '__name__'): + return cls.__name__ + return cls.name + + def disconnect_from_folder(self): + self.is_connected = False + + def is_usb_connected(self, devices_on_system, debug=False, + only_presence=False): + return self.is_connected, self + + def open(self): + if self._main_prefix is None: + raise NotImplementedError() + return True + + def set_progress_reporter(self, report_progress): + self.report_progress = report_progress + + def card_prefix(self, end_session=True): + return (None, None) + + def total_space(self, end_session=True): + return (1024*1024*1024, 0, 0) + + def free_space(self, end_session=True): + return (1024*1024*1024, 0, 0) + + def get_main_ebook_dir(self): + return '' diff --git a/src/calibre/devices/htc_td2/driver.py b/src/calibre/devices/htc_td2/driver.py index 9a83e32961..41eccfa0b2 100644 --- a/src/calibre/devices/htc_td2/driver.py +++ b/src/calibre/devices/htc_td2/driver.py @@ -19,7 +19,8 @@ class HTC_TD2(USBMS): VENDOR_ID = { # HTC - 0x0bb4 : { 0x0c30 : [0x000]}, +# 0x0bb4 : { 0x0c30 : [0x000]}, + 0xFbb4 : { 0x0c30 : [0x000]}, } EBOOK_DIR_MAIN = ['EBooks'] EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index af314c5468..048e5b0ccb 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -25,6 +25,7 @@ from calibre.utils.filenames import ascii_filename from calibre.devices.errors import FreeSpaceError from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config +from calibre.devices.folder_device.driver import FOLDER_DEVICE class DeviceJob(BaseJob): @@ -207,6 +208,27 @@ class DeviceManager(Thread): return self.create_job(self._get_device_information, done, description=_('Get device information')) + def connect_to_folder(self, path): + dev = FOLDER_DEVICE(path) + try: + dev.open() + except: + print 'Unable to open device', dev + traceback.print_exc() + return False + self.connected_device = dev + self.connected_slot(True) + return True + + def disconnect_folder(self): + if self.connected_device is not None: + if hasattr(self.connected_device, 'disconnect_from_folder'): + self.connected_device.disconnect_from_folder() + +# def connect_to_folder(self, path): +# return self.create_job(self._connect_to_folder, None, +# description=_('Connect to folder')) + def _books(self): '''Get metadata from device''' mainlist = self.device.books(oncard=None, end_session=False) @@ -309,6 +331,8 @@ class DeviceAction(QAction): class DeviceMenu(QMenu): fetch_annotations = pyqtSignal() + connect_to_folder = pyqtSignal() + disconnect_from_folder = pyqtSignal() def __init__(self, parent=None): QMenu.__init__(self, parent) @@ -410,6 +434,17 @@ class DeviceMenu(QMenu): annot.triggered.connect(lambda x : self.fetch_annotations.emit()) self.annotation_action = annot + + mitem = self.addAction(_('Connect to folder (experimental)')) + mitem.setEnabled(True) + mitem.triggered.connect(lambda x : self.connect_to_folder.emit()) + self.connect_to_folder_action = mitem + + mitem = self.addAction(_('Disconnect from folder (experimental)')) + mitem.setEnabled(False) + mitem.triggered.connect(lambda x : self.disconnect_from_folder.emit()) + self.disconnect_from_folder_action = mitem + self.enable_device_actions(False) def change_default_action(self, action): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index ba6bac76e4..8cd89bd397 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -666,6 +666,18 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) + def connect_to_folder(self): + dir = choose_dir(self, 'Select Device Folder', 'Select folder to open') + if dir is not None: + print dir + self.device_manager.connect_to_folder(dir) + self._sync_menu.connect_to_folder_action.setEnabled(False) + self._sync_menu.disconnect_from_folder_action.setEnabled(True) + + def disconnect_from_folder(self): + self.device_manager.disconnect_folder() + self._sync_menu.connect_to_folder_action.setEnabled(True) + self._sync_menu.disconnect_from_folder_action.setEnabled(False) def create_device_menu(self): self._sync_menu = DeviceMenu(self) @@ -676,6 +688,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.connect(self.action_sync, SIGNAL('triggered(bool)'), self._sync_menu.trigger_default) self._sync_menu.fetch_annotations.connect(self.fetch_annotations) + self._sync_menu.connect_to_folder.connect(self.connect_to_folder) + self._sync_menu.disconnect_from_folder.connect(self.disconnect_from_folder) def add_spare_server(self, *args): self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0))) From 583f9c1197491c325972fc79b1cb0601cd25e2e2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 20:35:15 +0100 Subject: [PATCH 10/20] Normalize paths for folder_device. --- src/calibre/devices/usbms/driver.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 95b7441f44..c6320f2746 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -174,11 +174,19 @@ class USBMS(CLI, Device): blist = 2 if location[1] == 'cardb' else 1 if location[1] == 'carda' else 0 if self._main_prefix: + # Normalize path and prefix + if self._main_prefix.find('\\') >= 0: + path = path.replace('/', '\\') + else: + path = path.replace('\\', '/') prefix = self._main_prefix if path.startswith(self._main_prefix) else None if not prefix and self._card_a_prefix: prefix = self._card_a_prefix if path.startswith(self._card_a_prefix) else None if not prefix and self._card_b_prefix: prefix = self._card_b_prefix if path.startswith(self._card_b_prefix) else None + if prefix is None: + print 'in add_books_to_metadata. Prefix is None!', path, self._main_prefix + continue lpath = path.partition(prefix)[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] From 3cfb28f0fff303b7f44aeacbce6d12343049b332 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 21:06:11 +0100 Subject: [PATCH 11/20] Regenerate sony_id_cache in fix_ids, because it changes all the values. --- src/calibre/devices/prs505/books.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 40a98913be..20fed3e2ed 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -357,6 +357,11 @@ def fix_ids(main, carda, cardb): item.parentNode.removeChild(item) item.unlink() db.reorder_playlists() + db.sony_id_cache = {} + for child in db.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + db.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + regen_ids(main) regen_ids(carda) From 0ce1a052b29417ba728b63a2f4e791361b84298b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 21:18:38 +0100 Subject: [PATCH 12/20] 1) don't try to sync if device is no longer connected 2) disable folder_device when another device is connected --- src/calibre/gui2/device.py | 3 ++- src/calibre/gui2/ui.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 048e5b0ccb..1703e4a644 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1135,4 +1135,5 @@ class DeviceGUI(object): resend_metadata = True if resend_metadata: # Correcting metadata cache on device. - self.device_manager.sync_booklists(None, booklists) + if self.device_manager.is_connected: + self.device_manager.sync_booklists(None, booklists) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 8cd89bd397..725672324c 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -948,6 +948,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): Called when a device is connected to the computer. ''' if connected: + self._sync_menu.connect_to_folder_action.setEnabled(False) + self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.device_manager.get_device_information(\ Dispatcher(self.info_read)) self.set_default_thumbnail(\ @@ -963,6 +965,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.eject_action.setEnabled(True) # don't refresh_ondevice here. It will happen in metadata_downloaded else: + self._sync_menu.connect_to_folder_action.setEnabled(True) + self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.save_device_view_settings() self.device_connected = False self._sync_menu.enable_device_actions(False) From 8b197ebd660718ac955e368662cb531f086d5e47 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 15 May 2010 22:13:30 +0100 Subject: [PATCH 13/20] 1) clean up refreshing ondevice when devices are plugged in and out. 2) close xml_file in sony BookList.init --- src/calibre/devices/prs505/books.py | 6 +++--- src/calibre/gui2/ui.py | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 20fed3e2ed..61f3e3c363 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -41,9 +41,9 @@ class BookList(_BookList): opts = settings() self.collections = opts.extra_customization.split(',') if opts.extra_customization else [] db = CACHE_XML if oncard else MEDIA_XML - xml_file = open(prefix + db, 'rb') - xml_file.seek(0) - self.document = dom.parse(xml_file) + with open(prefix + db, 'rb') as xml_file: + xml_file.seek(0) + self.document = dom.parse(xml_file) self.root_element = self.document.documentElement self.mountpath = prefix records = self.root_element.getElementsByTagName('records') diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 725672324c..9bb89dec68 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -949,7 +949,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ''' if connected: self._sync_menu.connect_to_folder_action.setEnabled(False) - self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.device_manager.get_device_information(\ Dispatcher(self.info_read)) self.set_default_thumbnail(\ @@ -963,10 +962,9 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_manager.device) self.location_view.model().device_connected(self.device_manager.device) self.eject_action.setEnabled(True) - # don't refresh_ondevice here. It will happen in metadata_downloaded + self.refresh_ondevice_info (device_connected = True, reset_only = True) else: self._sync_menu.connect_to_folder_action.setEnabled(True) - self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.save_device_view_settings() self.device_connected = False self._sync_menu.enable_device_actions(False) @@ -1035,10 +1033,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ############################################################################ ### Force the library view to refresh, taking into consideration books information - def refresh_ondevice_info(self, device_connected): - # Save current column widths because we might be turning on OnDevice - self.library_view.write_settings() + def refresh_ondevice_info(self, device_connected, reset_only = False): self.book_on_device(None, reset=True) + if reset_only: + return + self.library_view.write_settings() self.library_view.model().set_device_connected(device_connected) ############################################################################ From 70a3207906dd91e3142e697899f0718b32daabc3 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 09:21:41 +0100 Subject: [PATCH 14/20] 1) folder device fixes 2) added configuration of the folder device 3) fixed set_books_in_library to save/restore search state, necessary because it must scan the entire database, not just the search results. 4) removed the HTC driver --- src/calibre/customize/builtins.py | 4 +- src/calibre/devices/folder_device/driver.py | 51 +++++++++++++-------- src/calibre/devices/htc_td2/__init__.py | 10 ---- src/calibre/devices/htc_td2/driver.py | 45 ------------------ src/calibre/devices/usbms/device.py | 6 ++- src/calibre/gui2/device.py | 49 ++++++++++++-------- src/calibre/gui2/ui.py | 5 +- src/calibre/library/caches.py | 8 ++++ src/calibre/library/database2.py | 2 + 9 files changed, 78 insertions(+), 102 deletions(-) delete mode 100644 src/calibre/devices/htc_td2/__init__.py delete mode 100644 src/calibre/devices/htc_td2/driver.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 1ad6c03fc2..6865954440 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -455,7 +455,7 @@ from calibre.devices.edge.driver import EDGE from calibre.devices.teclast.driver import TECLAST_K3 from calibre.devices.sne.driver import SNE from calibre.devices.misc import PALMPRE, KOBO, AVANT -from calibre.devices.htc_td2.driver import HTC_TD2 +from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon from calibre.library.catalog import CSV_XML, EPUB_MOBI @@ -540,7 +540,7 @@ plugins += [ PALMPRE, KOBO, AZBOOKA, - HTC_TD2, + FOLDER_DEVICE_FOR_CONFIG, AVANT, ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 700b7f3eec..31da69d49a 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -4,36 +4,48 @@ Created on 15 May 2010 @author: charles ''' import os -import time -from calibre.customize.ui import available_output_formats from calibre.devices.usbms.driver import USBMS, BookList -from calibre.devices.interface import DevicePlugin -from calibre.devices.usbms.deviceconfig import DeviceConfig -from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to + +# This class is added to the standard device plugin chain, so that it can +# be configured. It has invalid vendor_id etc, so it will never match a +# device. The 'real' FOLDER_DEVICE will use the config from it. +class FOLDER_DEVICE_FOR_CONFIG(USBMS): + name = 'Folder Device Interface' + gui_name = 'Folder Device' + description = _('Use an arbitrary folder as a device.') + author = 'John Schember/Charles Haley' + supported_platforms = ['windows', 'osx', 'linux'] + FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] class FOLDER_DEVICE(USBMS): type = _('Device Interface') - # Ordered list of supported formats + name = 'Folder Device Interface' + gui_name = 'Folder Device' + description = _('Use an arbitrary folder as a device.') + author = 'John Schember/Charles Haley' + supported_platforms = ['windows', 'osx', 'linux'] FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device - # Whether the metadata on books can be set via the GUI. + CAN_SET_METADATA = True SUPPORTS_SUB_DIRS = True - DELETE_EXTS = [] - #: Path separator for paths to books on device - path_sep = os.sep + #: Icon for this device - icon = I('reader.svg') + icon = I('sd.svg') METADATA_CACHE = '.metadata.calibre' - _main_prefix = None + _main_prefix = '' _card_a_prefix = None _card_b_prefix = None + is_connected = False + def __init__(self, path): + if not os.path.isdir(path): + raise IOError, 'Path is not a folder' self._main_prefix = path self.booklist_class = BookList self.is_connected = True @@ -47,6 +59,7 @@ class FOLDER_DEVICE(USBMS): return cls.name def disconnect_from_folder(self): + self._main_prefix = '' self.is_connected = False def is_usb_connected(self, devices_on_system, debug=False, @@ -54,8 +67,8 @@ class FOLDER_DEVICE(USBMS): return self.is_connected, self def open(self): - if self._main_prefix is None: - raise NotImplementedError() + if not self._main_prefix: + return False return True def set_progress_reporter(self, report_progress): @@ -64,11 +77,9 @@ class FOLDER_DEVICE(USBMS): def card_prefix(self, end_session=True): return (None, None) - def total_space(self, end_session=True): - return (1024*1024*1024, 0, 0) - - def free_space(self, end_session=True): - return (1024*1024*1024, 0, 0) - def get_main_ebook_dir(self): return '' + + @classmethod + def settings(self): + return FOLDER_DEVICE_FOR_CONFIG._config().parse() diff --git a/src/calibre/devices/htc_td2/__init__.py b/src/calibre/devices/htc_td2/__init__.py deleted file mode 100644 index 3d1a86922e..0000000000 --- a/src/calibre/devices/htc_td2/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - - - diff --git a/src/calibre/devices/htc_td2/driver.py b/src/calibre/devices/htc_td2/driver.py deleted file mode 100644 index 41eccfa0b2..0000000000 --- a/src/calibre/devices/htc_td2/driver.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -from calibre.devices.usbms.driver import USBMS - -class HTC_TD2(USBMS): - - name = 'HTC TD2 Phone driver' - gui_name = 'HTC TD2' - description = _('Communicate with HTC TD2 phones.') - author = 'Charles Haley' - supported_platforms = ['osx', 'linux'] - - # Ordered list of supported formats - FORMATS = ['epub', 'pdf'] - - VENDOR_ID = { - # HTC -# 0x0bb4 : { 0x0c30 : [0x000]}, - 0xFbb4 : { 0x0c30 : [0x000]}, - } - EBOOK_DIR_MAIN = ['EBooks'] - EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' - 'send e-books to on the device. The first one that exists will ' - 'be used') - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN) - - VENDOR_NAME = [''] - WINDOWS_MAIN_MEM = [''] - - MAIN_MEMORY_VOLUME_LABEL = 'HTC Phone Internal Memory' - - SUPPORTS_SUB_DIRS = True - - def post_open_callback(self): - opts = self.settings() - dirs = opts.extra_customization - if not dirs: - dirs = self.EBOOK_DIR_MAIN - else: - dirs = [x.strip() for x in dirs.split(',')] - self.EBOOK_DIR_MAIN = dirs diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 1b048d1bb6..249733b4e3 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -113,15 +113,17 @@ class Device(DeviceConfig, DevicePlugin): def _windows_space(cls, prefix): if not prefix: return 0, 0 + if prefix.endswith(os.sep): + prefix = prefix[:-1] win32file = __import__('win32file', globals(), locals(), [], -1) try: sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \ - win32file.GetDiskFreeSpace(prefix[:-1]) + win32file.GetDiskFreeSpace(prefix) except Exception, err: if getattr(err, 'args', [None])[0] == 21: # Disk not ready time.sleep(3) sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \ - win32file.GetDiskFreeSpace(prefix[:-1]) + win32file.GetDiskFreeSpace(prefix) else: raise mult = sectors_per_cluster * bytes_per_sector return total_clusters * mult, free_clusters * mult diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 1703e4a644..d6f1a7a205 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -428,23 +428,24 @@ class DeviceMenu(QMenu): if opts.accounts: self.addSeparator() self.addMenu(self.email_to_menu) + + self.addSeparator() + mitem = self.addAction(_('Connect to folder')) + mitem.setEnabled(True) + mitem.triggered.connect(lambda x : self.connect_to_folder.emit()) + self.connect_to_folder_action = mitem + + mitem = self.addAction(_('Disconnect from folder')) + mitem.setEnabled(False) + mitem.triggered.connect(lambda x : self.disconnect_from_folder.emit()) + self.disconnect_from_folder_action = mitem + self.addSeparator() annot = self.addAction(_('Fetch annotations (experimental)')) annot.setEnabled(False) annot.triggered.connect(lambda x : self.fetch_annotations.emit()) self.annotation_action = annot - - mitem = self.addAction(_('Connect to folder (experimental)')) - mitem.setEnabled(True) - mitem.triggered.connect(lambda x : self.connect_to_folder.emit()) - self.connect_to_folder_action = mitem - - mitem = self.addAction(_('Disconnect from folder (experimental)')) - mitem.setEnabled(False) - mitem.triggered.connect(lambda x : self.disconnect_from_folder.emit()) - self.disconnect_from_folder_action = mitem - self.enable_device_actions(False) def change_default_action(self, action): @@ -1089,8 +1090,17 @@ class DeviceGUI(object): # First build a cache of the library, so the search isn't On**2 self.db_book_title_cache = {} self.db_book_uuid_cache = set() - for idx in range(self.library_view.model().db.count()): - mi = self.library_view.model().db.get_metadata(idx, index_is_id=False) + db = self.library_view.model().db + # The following is a terrible hack, made necessary because the db + # result_cache will always use the results filtered by the current + # search. We need all the db entries here. Choice was to either + # cache the search results so we can use the entire db, to duplicate + # large parts of the get_metadata code, or to use db_ids and pay the + # large performance penalty of zillions of SQL queries. Choice: + # save/restore the search state. + state = db.get_state_before_scan() + for idx in range(db.count()): + mi = db.get_metadata(idx, index_is_id=False) title = re.sub('(?u)\W|[_]', '', mi.title.lower()) if title not in self.db_book_title_cache: self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}} @@ -1099,12 +1109,13 @@ class DeviceGUI(object): self.db_book_title_cache[title]['authors'][authors] = mi self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi self.db_book_uuid_cache.add(mi.uuid) + db.restore_state_after_scan(state) - # Now iterate through all the books on the device, setting the in_library field - # Fastest and most accurate key is the uuid. Second is the application_id, which - # is really the db key, but as this can accidentally match across libraries we - # also verify the title. The db_id exists on Sony devices. Fallback is title - # and author match + # Now iterate through all the books on the device, setting the + # in_library field Fastest and most accurate key is the uuid. Second is + # the application_id, which is really the db key, but as this can + # accidentally match across libraries we also verify the title. The + # db_id exists on Sony devices. Fallback is title and author match resend_metadata = False for booklist in booklists: for book in booklist: @@ -1135,5 +1146,5 @@ class DeviceGUI(object): resend_metadata = True if resend_metadata: # Correcting metadata cache on device. - if self.device_manager.is_connected: + if self.device_manager.is_device_connected: self.device_manager.sync_booklists(None, booklists) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9bb89dec68..23a0490f14 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -669,15 +669,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): def connect_to_folder(self): dir = choose_dir(self, 'Select Device Folder', 'Select folder to open') if dir is not None: - print dir self.device_manager.connect_to_folder(dir) - self._sync_menu.connect_to_folder_action.setEnabled(False) self._sync_menu.disconnect_from_folder_action.setEnabled(True) def disconnect_from_folder(self): self.device_manager.disconnect_folder() - self._sync_menu.connect_to_folder_action.setEnabled(True) - self._sync_menu.disconnect_from_folder_action.setEnabled(False) def create_device_menu(self): self._sync_menu = DeviceMenu(self) @@ -965,6 +961,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.refresh_ondevice_info (device_connected = True, reset_only = True) else: self._sync_menu.connect_to_folder_action.setEnabled(True) + self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.save_device_view_settings() self.device_connected = False self._sync_menu.enable_device_actions(False) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index acc8eaffb6..73faa6f1ab 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -553,6 +553,14 @@ class ResultCache(SearchQueryParser): if item is not None: item[ondevice_col] = db.book_on_device_string(item[0]) + def get_state_before_scan(self): + retval = self._map_filtered + self._map_filtered = self._map + return retval + + def restore_state_after_scan(self, map_filtered): + self._map_filtered = map_filtered + def refresh(self, db, field=None, ascending=True): temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 5971333078..063538656f 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count + self.get_state_before_scan = self.data.get_state_before_scan + self.restore_state_after_scan = self.data.restore_state_after_scan self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() From a522f76a2154c71a898ac5ffb04f5041f5c2ce74 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 18:05:45 +0100 Subject: [PATCH 15/20] Commit for starson17 testing --- src/calibre/devices/usbms/driver.py | 46 ++++++++++++++++++----------- src/calibre/gui2/device.py | 4 +++ src/calibre/gui2/library.py | 9 +++++- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index c6320f2746..7a46ef3dc7 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -78,7 +78,7 @@ class USBMS(CLI, Device): changed = False if path_to_ext(filename) in self.FORMATS: try: - lpath = os.path.join(path, filename).partition(prefix)[2] + lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] idx = bl_cache.get(lpath.replace('\\', '/'), None) @@ -98,7 +98,9 @@ class USBMS(CLI, Device): if isinstance(ebook_dirs, basestring): ebook_dirs = [ebook_dirs] for ebook_dir in ebook_dirs: - ebook_dir = os.path.join(prefix, *(ebook_dir.split('/'))) if ebook_dir else prefix + ebook_dir = self.normalize_path( \ + os.path.join(prefix, *(ebook_dir.split('/'))) \ + if ebook_dir else prefix) if not os.path.exists(ebook_dir): continue # Get all books in the ebook_dir directory if self.SUPPORTS_SUB_DIRS: @@ -119,6 +121,7 @@ class USBMS(CLI, Device): # if count != len(bl) then there were items in it that we did not # find on the device. If need_sync is True then there were either items # on the device that were not in bl or some of the items were changed. + print "count found in cache: %d, count of files in cache: %d, must_sync_cache: %s" % (self.count_found_in_bl, len(bl), need_sync) if self.count_found_in_bl != len(bl) or need_sync: if oncard == 'cardb': self.sync_booklists((None, None, metadata)) @@ -140,13 +143,12 @@ class USBMS(CLI, Device): for i, infile in enumerate(files): mdata, fname = metadata.next(), names.next() - filepath = self.create_upload_path(path, mdata, fname) - + filepath = self.normalize_path(self.create_upload_path(path, mdata, fname)) paths.append(filepath) - - self.put_file(infile, filepath, replace_file=True) + self.put_file(self.normalize_path(infile), filepath, replace_file=True) try: - self.upload_cover(os.path.dirname(filepath), os.path.splitext(os.path.basename(filepath))[0], mdata) + self.upload_cover(os.path.dirname(filepath), + os.path.splitext(os.path.basename(filepath))[0], mdata) except: # Failure to upload cover is not catastrophic import traceback traceback.print_exc() @@ -192,14 +194,14 @@ class USBMS(CLI, Device): lpath = lpath[len(os.sep):] book = self.book_class(prefix, lpath, other=info) if book.size is None: - book.size = os.stat(path).st_size - + book.size = os.stat(self.normalize_path(path)).st_size booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) def delete_books(self, paths, end_session=True): for i, path in enumerate(paths): self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) + path = self.normalize_path(path) if os.path.exists(path): # Delete the ebook os.unlink(path) @@ -228,15 +230,15 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Removing books from device metadata listing...')) def sync_booklists(self, booklists, end_session=True): - if not os.path.exists(self._main_prefix): - os.makedirs(self._main_prefix) + if not os.path.exists(self.normalize_path(self._main_prefix)): + os.makedirs(self.normalize_path(self._main_prefix)) def write_prefix(prefix, listid): if prefix is not None and isinstance(booklists[listid], self.booklist_class): if not os.path.exists(prefix): - os.makedirs(prefix) + os.makedirs(self.normalize_path(prefix)) js = [item.to_json() for item in booklists[listid]] - with open(os.path.join(prefix, self.METADATA_CACHE), 'wb') as f: + with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: json.dump(js, f, indent=2, encoding='utf-8') write_prefix(self._main_prefix, 0) write_prefix(self._card_a_prefix, 1) @@ -244,13 +246,21 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) + @classmethod + def normalize_path(cls, path): + if os.sep == '\\': + path = path.replace('/', '\\') + else: + path = path.replace('\\', '/') + return path + @classmethod def parse_metadata_cache(cls, prefix, name): bl = [] js = [] need_sync = False try: - with open(os.path.join(prefix, name), 'rb') as f: + with open(cls.normalize_path(os.path.join(prefix, name)), 'rb') as f: js = json.load(f, encoding='utf-8') for item in js: book = cls.book_class(prefix, item.get('lpath', None)) @@ -267,7 +277,7 @@ class USBMS(CLI, Device): @classmethod def update_metadata_item(cls, item): changed = False - size = os.stat(item.path).st_size + size = os.stat(cls.normalize_path(item.path)).st_size if size != item.size: changed = True mi = cls.metadata_from_path(item.path) @@ -291,15 +301,15 @@ class USBMS(CLI, Device): from calibre.ebooks.metadata import MetaInformation if cls.settings().read_metadata or cls.MUST_READ_METADATA: - mi = cls.metadata_from_path(os.path.join(prefix, path)) + mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, path))) else: from calibre.ebooks.metadata.meta import metadata_from_filename - mi = metadata_from_filename(os.path.basename(path), + mi = metadata_from_filename(cls.normalize_path(os.path.basename(path)), re.compile(r'^(?P[ \S]+?)[ _]-[ _](?P<author>[ \S]+?)_+\d+')) if mi is None: mi = MetaInformation(os.path.splitext(os.path.basename(path))[0], [_('Unknown')]) - mi.size = os.stat(os.path.join(prefix, path)).st_size + mi.size = os.stat(cls.normalize_path(os.path.join(prefix, path))).st_size book = cls.book_class(prefix, path, other=mi) return book diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index d6f1a7a205..c5fdbec2dd 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1144,6 +1144,10 @@ class DeviceGUI(object): book.in_library = True book.smart_update(d['authors'][book_authors]) resend_metadata = True + # Set author_sort if it isn't already + asort = getattr(book, 'author_sort', None) + if not asort: + pass if resend_metadata: # Correcting metadata cache on device. if self.device_manager.is_device_connected: diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index eeda687312..9a9ffb5d94 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -1379,7 +1379,14 @@ class DeviceBooksModel(BooksModel): x, y = self.db[x].in_library, self.db[y].in_library return cmp(x, y) def authorcmp(x, y): - x, y = authors_to_string(self.db[x].authors), authors_to_string(self.db[y].authors) + ax = getattr(self.db[x], 'author_sort', None) + ay = getattr(self.db[y], 'author_sort', None) + if ax and ay: + x = ax + y = ay + else: + x, y = authors_to_string(self.db[x].authors), \ + authors_to_string(self.db[y].authors) return cmp(x, y) fcmp = strcmp('title_sorter') if col == 0 else authorcmp if col == 1 else \ sizecmp if col == 2 else datecmp if col == 3 else tagscmp if col == 4 else libcmp From 1e90881b905886ad584cca71df7454482cd7d0e7 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 18:56:56 +0100 Subject: [PATCH 16/20] 1) changes to ondevice scan to use Kovid's iterator 2) correction to path construction in USMBS --- src/calibre/devices/usbms/driver.py | 4 ++-- src/calibre/gui2/device.py | 13 ++----------- src/calibre/library/caches.py | 8 -------- src/calibre/library/database2.py | 2 -- 4 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 7a46ef3dc7..c5b3d653c3 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -190,8 +190,8 @@ class USBMS(CLI, Device): print 'in add_books_to_metadata. Prefix is None!', path, self._main_prefix continue lpath = path.partition(prefix)[2] - if lpath.startswith(os.sep): - lpath = lpath[len(os.sep):] + if lpath.startswith('/') or lpath.startswith('\\'): + lpath = lpath[1:] book = self.book_class(prefix, lpath, other=info) if book.size is None: book.size = os.stat(self.normalize_path(path)).st_size diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index c5fdbec2dd..fa77dc862f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1091,16 +1091,8 @@ class DeviceGUI(object): self.db_book_title_cache = {} self.db_book_uuid_cache = set() db = self.library_view.model().db - # The following is a terrible hack, made necessary because the db - # result_cache will always use the results filtered by the current - # search. We need all the db entries here. Choice was to either - # cache the search results so we can use the entire db, to duplicate - # large parts of the get_metadata code, or to use db_ids and pay the - # large performance penalty of zillions of SQL queries. Choice: - # save/restore the search state. - state = db.get_state_before_scan() - for idx in range(db.count()): - mi = db.get_metadata(idx, index_is_id=False) + for id in db.data.iterallids(): + mi = db.get_metadata(id, index_is_id=True) title = re.sub('(?u)\W|[_]', '', mi.title.lower()) if title not in self.db_book_title_cache: self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}} @@ -1109,7 +1101,6 @@ class DeviceGUI(object): self.db_book_title_cache[title]['authors'][authors] = mi self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi self.db_book_uuid_cache.add(mi.uuid) - db.restore_state_after_scan(state) # Now iterate through all the books on the device, setting the # in_library field Fastest and most accurate key is the uuid. Second is diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 45a357c1b5..e280a2178b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -563,14 +563,6 @@ class ResultCache(SearchQueryParser): if item is not None: item[ondevice_col] = db.book_on_device_string(item[0]) - def get_state_before_scan(self): - retval = self._map_filtered - self._map_filtered = self._map - return retval - - def restore_state_after_scan(self, map_filtered): - self._map_filtered = map_filtered - def refresh(self, db, field=None, ascending=True): temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 063538656f..5971333078 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -245,8 +245,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count - self.get_state_before_scan = self.data.get_state_before_scan - self.restore_state_after_scan = self.data.restore_state_after_scan self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() From 0a4dd08686553e162d112f84a244f8227b22cfa2 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 20:37:49 +0100 Subject: [PATCH 17/20] Use an invalid vendor ID for the folder_device --- src/calibre/devices/folder_device/driver.py | 4 ++++ src/calibre/gui2/device.py | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 31da69d49a..2b4fc4dea9 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -28,6 +28,10 @@ class FOLDER_DEVICE(USBMS): supported_platforms = ['windows', 'osx', 'linux'] FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] + VENDOR_ID = 0xffff + PRODUCT_ID = 0xffff + BCD = 0xffff + THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device CAN_SET_METADATA = True diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index fa77dc862f..31fe4bbbbd 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -225,10 +225,6 @@ class DeviceManager(Thread): if hasattr(self.connected_device, 'disconnect_from_folder'): self.connected_device.disconnect_from_folder() -# def connect_to_folder(self, path): -# return self.create_job(self._connect_to_folder, None, -# description=_('Connect to folder')) - def _books(self): '''Get metadata from device''' mainlist = self.device.books(oncard=None, end_session=False) From aa36a2aada2793b62c1a9c1511602efdf021dadd Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 20:38:30 +0100 Subject: [PATCH 18/20] Put the vendor ID in the right place. --- src/calibre/devices/folder_device/driver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 2b4fc4dea9..e7d09675c7 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -17,6 +17,10 @@ class FOLDER_DEVICE_FOR_CONFIG(USBMS): author = 'John Schember/Charles Haley' supported_platforms = ['windows', 'osx', 'linux'] FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] + VENDOR_ID = 0xffff + PRODUCT_ID = 0xffff + BCD = 0xffff + class FOLDER_DEVICE(USBMS): type = _('Device Interface') From 462ae5b9e24145943ac70f3441477b49285da56c Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 21:13:34 +0100 Subject: [PATCH 19/20] Clean up eject_device --- src/calibre/devices/folder_device/driver.py | 3 +++ src/calibre/devices/usbms/driver.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index e7d09675c7..f85fca55e1 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -88,6 +88,9 @@ class FOLDER_DEVICE(USBMS): def get_main_ebook_dir(self): return '' + def eject(self): + self.is_connected = False + @classmethod def settings(self): return FOLDER_DEVICE_FOR_CONFIG._config().parse() diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index c5b3d653c3..332f337a2f 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -248,6 +248,8 @@ class USBMS(CLI, Device): @classmethod def normalize_path(cls, path): + if path is None: + return None if os.sep == '\\': path = path.replace('/', '\\') else: From 92753c6f978965671005bcef32968c2a085f4857 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 16 May 2010 22:12:16 +0100 Subject: [PATCH 20/20] Add confirmation dialog to delete from device --- src/calibre/gui2/ui.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 5343583f5c..c1e208625b 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -1523,6 +1523,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): sm = view.selectionModel() sm.select(ci, sm.Select) else: + if not confirm('<p>'+_('The selected books will be ' + '<b>permanently deleted</b> ' + 'from your device. Are you sure?') + +'</p>', 'library_delete_books', self): + return if self.stack.currentIndex() == 1: view = self.memory_view elif self.stack.currentIndex() == 2: