diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 7cc5713eda..17239256df 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -455,7 +455,7 @@ from calibre.devices.edge.driver import EDGE from calibre.devices.teclast.driver import TECLAST_K3 from calibre.devices.sne.driver import SNE from calibre.devices.misc import PALMPRE, KOBO, AVANT -from calibre.devices.htc_td2.driver import HTC_TD2 +from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon from calibre.library.catalog import CSV_XML, EPUB_MOBI @@ -540,7 +540,7 @@ plugins += [ PALMPRE, KOBO, AZBOOKA, - HTC_TD2, + FOLDER_DEVICE_FOR_CONFIG, AVANT, ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ diff --git a/src/calibre/devices/htc_td2/__init__.py b/src/calibre/devices/folder_device/__init__.py similarity index 100% rename from src/calibre/devices/htc_td2/__init__.py rename to src/calibre/devices/folder_device/__init__.py diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py new file mode 100644 index 0000000000..f85fca55e1 --- /dev/null +++ b/src/calibre/devices/folder_device/driver.py @@ -0,0 +1,96 @@ +''' +Created on 15 May 2010 + +@author: charles +''' +import os + +from calibre.devices.usbms.driver import USBMS, BookList + +# This class is added to the standard device plugin chain, so that it can +# be configured. It has invalid vendor_id etc, so it will never match a +# device. The 'real' FOLDER_DEVICE will use the config from it. +class FOLDER_DEVICE_FOR_CONFIG(USBMS): + name = 'Folder Device Interface' + gui_name = 'Folder Device' + description = _('Use an arbitrary folder as a device.') + author = 'John Schember/Charles Haley' + supported_platforms = ['windows', 'osx', 'linux'] + FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] + VENDOR_ID = 0xffff + PRODUCT_ID = 0xffff + BCD = 0xffff + + +class FOLDER_DEVICE(USBMS): + type = _('Device Interface') + + name = 'Folder Device Interface' + gui_name = 'Folder Device' + description = _('Use an arbitrary folder as a device.') + author = 'John Schember/Charles Haley' + supported_platforms = ['windows', 'osx', 'linux'] + FORMATS = ['epub', 'fb2', 'mobi', 'lrf', 'tcr', 'pmlz', 'lit', 'rtf', 'rb', 'pdf', 'oeb', 'txt', 'pdb'] + + VENDOR_ID = 0xffff + PRODUCT_ID = 0xffff + BCD = 0xffff + + THUMBNAIL_HEIGHT = 68 # Height for thumbnails on device + + CAN_SET_METADATA = True + SUPPORTS_SUB_DIRS = True + + #: Icon for this device + icon = I('sd.svg') + METADATA_CACHE = '.metadata.calibre' + + _main_prefix = '' + _card_a_prefix = None + _card_b_prefix = None + + is_connected = False + + def __init__(self, path): + if not os.path.isdir(path): + raise IOError, 'Path is not a folder' + self._main_prefix = path + self.booklist_class = BookList + self.is_connected = True + + @classmethod + def get_gui_name(cls): + if hasattr(cls, 'gui_name'): + return cls.gui_name + if hasattr(cls, '__name__'): + return cls.__name__ + return cls.name + + def disconnect_from_folder(self): + self._main_prefix = '' + self.is_connected = False + + def is_usb_connected(self, devices_on_system, debug=False, + only_presence=False): + return self.is_connected, self + + def open(self): + if not self._main_prefix: + return False + return True + + def set_progress_reporter(self, report_progress): + self.report_progress = report_progress + + def card_prefix(self, end_session=True): + return (None, None) + + def get_main_ebook_dir(self): + return '' + + def eject(self): + self.is_connected = False + + @classmethod + def settings(self): + return FOLDER_DEVICE_FOR_CONFIG._config().parse() diff --git a/src/calibre/devices/htc_td2/driver.py b/src/calibre/devices/htc_td2/driver.py deleted file mode 100644 index 9a83e32961..0000000000 --- a/src/calibre/devices/htc_td2/driver.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -from calibre.devices.usbms.driver import USBMS - -class HTC_TD2(USBMS): - - name = 'HTC TD2 Phone driver' - gui_name = 'HTC TD2' - description = _('Communicate with HTC TD2 phones.') - author = 'Charles Haley' - supported_platforms = ['osx', 'linux'] - - # Ordered list of supported formats - FORMATS = ['epub', 'pdf'] - - VENDOR_ID = { - # HTC - 0x0bb4 : { 0x0c30 : [0x000]}, - } - EBOOK_DIR_MAIN = ['EBooks'] - EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of directories to ' - 'send e-books to on the device. The first one that exists will ' - 'be used') - EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(EBOOK_DIR_MAIN) - - VENDOR_NAME = [''] - WINDOWS_MAIN_MEM = [''] - - MAIN_MEMORY_VOLUME_LABEL = 'HTC Phone Internal Memory' - - SUPPORTS_SUB_DIRS = True - - def post_open_callback(self): - opts = self.settings() - dirs = opts.extra_customization - if not dirs: - dirs = self.EBOOK_DIR_MAIN - else: - dirs = [x.strip() for x in dirs.split(',')] - self.EBOOK_DIR_MAIN = dirs diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 98421959cc..b38b62e20c 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -387,6 +387,9 @@ class BookList(list): __getslice__ = None __setslice__ = None + def __init__(self, oncard, prefix): + pass + def supports_tags(self): ''' Return True if the the device supports tags (collections) for this book list. ''' raise NotImplementedError() diff --git a/src/calibre/devices/prs505/__init__.py b/src/calibre/devices/prs505/__init__.py index f832dbb7fc..20f3b8d49b 100644 --- a/src/calibre/devices/prs505/__init__.py +++ b/src/calibre/devices/prs505/__init__.py @@ -1,2 +1,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' + +MEDIA_XML = 'database/cache/media.xml' + +CACHE_XML = 'Sony Reader/database/cache.xml' diff --git a/src/calibre/devices/prs505/books.py b/src/calibre/devices/prs505/books.py index 66f24b97a0..61f3e3c363 100644 --- a/src/calibre/devices/prs505/books.py +++ b/src/calibre/devices/prs505/books.py @@ -5,13 +5,13 @@ __copyright__ = '2008, Kovid Goyal ' import re, time, functools from uuid import uuid4 as _uuid import xml.dom.minidom as dom -from base64 import b64decode as decode from base64 import b64encode as encode -from calibre.devices.interface import BookList as _BookList +from calibre.devices.usbms.books import BookList as _BookList from calibre.devices import strftime as _strftime -from calibre.devices import strptime +from calibre.devices.prs505 import MEDIA_XML, CACHE_XML +from calibre.devices.errors import PathError strftime = functools.partial(_strftime, zone=time.gmtime) @@ -30,127 +30,43 @@ def uuid(): def sortable_title(title): return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip() -class book_metadata_field(object): - """ Represents metadata stored as an attribute """ - def __init__(self, attr, formatter=None, setter=None): - self.attr = attr - self.formatter = formatter - self.setter = setter - - def __get__(self, obj, typ=None): - """ Return a string. String may be empty if self.attr is absent """ - return self.formatter(obj.elem.getAttribute(self.attr)) if \ - self.formatter else obj.elem.getAttribute(self.attr).strip() - - def __set__(self, obj, val): - """ Set the attribute """ - val = self.setter(val) if self.setter else val - if not isinstance(val, unicode): - val = unicode(val, 'utf8', 'replace') - obj.elem.setAttribute(self.attr, val) - - -class Book(object): - """ Provides a view onto the XML element that represents a book """ - - title = book_metadata_field("title") - authors = book_metadata_field("author", \ - formatter=lambda x: [x if x and x.strip() else _('Unknown')]) - mime = book_metadata_field("mime") - rpath = book_metadata_field("path") - id = book_metadata_field("id", formatter=int) - sourceid = book_metadata_field("sourceid", formatter=int) - size = book_metadata_field("size", formatter=lambda x : int(float(x))) - # When setting this attribute you must use an epoch - datetime = book_metadata_field("date", formatter=strptime, setter=strftime) - - @dynamic_property - def title_sorter(self): - doc = '''String to sort the title. If absent, title is returned''' - def fget(self): - src = self.elem.getAttribute('titleSorter').strip() - if not src: - src = self.title - return src - def fset(self, val): - self.elem.setAttribute('titleSorter', sortable_title(unicode(val))) - return property(doc=doc, fget=fget, fset=fset) - - @dynamic_property - def thumbnail(self): - doc = \ - """ - The thumbnail. Should be a height 68 image. - Setting is not supported. - """ - def fget(self): - th = self.elem.getElementsByTagName(self.prefix + "thumbnail") - if not len(th): - th = self.elem.getElementsByTagName("cache:thumbnail") - if len(th): - for n in th[0].childNodes: - if n.nodeType == n.ELEMENT_NODE: - th = n - break - rc = "" - for node in th.childNodes: - if node.nodeType == node.TEXT_NODE: - rc += node.data - return decode(rc) - return property(fget=fget, doc=doc) - - @dynamic_property - def path(self): - doc = """ Absolute path to book on device. Setting not supported. """ - def fget(self): - return self.mountpath + self.rpath - return property(fget=fget, doc=doc) - - @dynamic_property - def db_id(self): - doc = '''The database id in the application database that this file corresponds to''' - def fget(self): - match = re.search(r'_(\d+)$', self.rpath.rpartition('.')[0]) - if match: - return int(match.group(1)) - return property(fget=fget, doc=doc) - - def __init__(self, node, mountpath, tags, prefix=""): - self.elem = node - self.prefix = prefix - self.tags = tags - self.mountpath = mountpath - - def __str__(self): - """ Return a utf-8 encoded string with title author and path information """ - return self.title.encode('utf-8') + " by " + \ - self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') - - class BookList(_BookList): - def __init__(self, xml_file, mountpath, report_progress=None): - _BookList.__init__(self) - xml_file.seek(0) - self.document = dom.parse(xml_file) + def __init__(self, oncard, prefix, settings): + _BookList.__init__(self, oncard, prefix, settings) + if prefix is None: + return + self.sony_id_cache = {} + self.books_lpath_cache = {} + opts = settings() + self.collections = opts.extra_customization.split(',') if opts.extra_customization else [] + db = CACHE_XML if oncard else MEDIA_XML + with open(prefix + db, 'rb') as xml_file: + xml_file.seek(0) + self.document = dom.parse(xml_file) self.root_element = self.document.documentElement - self.mountpath = mountpath + self.mountpath = prefix records = self.root_element.getElementsByTagName('records') - self.tag_order = {} if records: self.prefix = 'xs1:' self.root_element = records[0] else: self.prefix = '' + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + self.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + # set the key to none. Will be filled in later when booklist is built + self.books_lpath_cache[child.getAttribute('path')] = None + self.tag_order = {} + + paths = self.purge_corrupted_files() + for path in paths: + try: + self.del_file(path, end_session=False) + except PathError: # Incase this is a refetch without a sync in between + continue - nodes = self.root_element.childNodes - for i, book in enumerate(nodes): - if report_progress: - report_progress((i+1) / float(len(nodes)), _('Getting list of books on device...')) - if hasattr(book, 'tagName') and book.tagName.endswith('text'): - tags = [i.getAttribute('title') for i in self.get_playlists(book.getAttribute('id'))] - self.append(Book(book, mountpath, tags, prefix=self.prefix)) def max_id(self): max = 0 @@ -173,39 +89,44 @@ class BookList(_BookList): def supports_tags(self): return True - def book_by_path(self, path): - for child in self.root_element.childNodes: - if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("path"): - if path == child.getAttribute('path'): - return child - return None - - def add_book(self, mi, name, collections, size, ctime): - """ Add a node into the DOM tree, representing a book """ - book = self.book_by_path(name) - if book is not None: - self.remove_book(name) - - node = self.document.createElement(self.prefix + "text") - mime = MIME_MAP.get(name.rpartition('.')[-1].lower(), MIME_MAP['epub']) + def add_book(self, book, replace_metadata): + # Add a node into the DOM tree, representing a book. Also add to booklist + if book in self: + # replacing metadata for book + self.delete_node(book.lpath) + else: + self.append(book) + if not replace_metadata: + if self.books_lpath_cache.has_key(book.lpath): + self.books_lpath_cache[book.lpath] = book + return + # Book not in metadata. Add it. Note that we don't need to worry about + # extra books in the Sony metadata. The reader deletes them for us when + # we disconnect. That said, if it becomes important one day, we can do + # it by scanning the books_lpath_cache for None entries and removing the + # corresponding nodes. + self.books_lpath_cache[book.lpath] = book cid = self.max_id()+1 + node = self.document.createElement(self.prefix + "text") + self.sony_id_cache[cid] = book.lpath + mime = MIME_MAP.get(book.lpath.rpartition('.')[-1].lower(), MIME_MAP['epub']) try: sourceid = str(self[0].sourceid) if len(self) else '1' except: sourceid = '1' attrs = { - "title" : mi.title, - 'titleSorter' : sortable_title(mi.title), - "author" : mi.format_authors() if mi.format_authors() else _('Unknown'), + "title" : book.title, + 'titleSorter' : sortable_title(book.title), + "author" : book.format_authors() if book.format_authors() else _('Unknown'), "page":"0", "part":"0", "scale":"0", \ "sourceid":sourceid, "id":str(cid), "date":"", \ - "mime":mime, "path":name, "size":str(size) + "mime":mime, "path":book.lpath, "size":str(book.size) } for attr in attrs.keys(): node.setAttributeNode(self.document.createAttribute(attr)) node.setAttribute(attr, attrs[attr]) try: - w, h, data = mi.thumbnail + w, h, data = book.thumbnail except: w, h, data = None, None, None @@ -218,14 +139,11 @@ class BookList(_BookList): th.appendChild(jpeg) node.appendChild(th) self.root_element.appendChild(node) - book = Book(node, self.mountpath, [], prefix=self.prefix) - book.datetime = ctime - self.append(book) tags = [] - for item in collections: + for item in self.collections: item = item.strip() - mitem = getattr(mi, item, None) + mitem = getattr(book, item, None) titems = [] if mitem: if isinstance(mitem, list): @@ -241,37 +159,36 @@ class BookList(_BookList): tags.extend(titems) if tags: tags = list(set(tags)) - if hasattr(mi, 'tag_order'): - self.tag_order.update(mi.tag_order) - self.set_tags(book, tags) + if hasattr(book, 'tag_order'): + self.tag_order.update(book.tag_order) + self.set_playlists(cid, tags) + return True # metadata cache has changed. Must sync at end - def _delete_book(self, node): + def _delete_node(self, node): nid = node.getAttribute('id') self.remove_from_playlists(nid) node.parentNode.removeChild(node) node.unlink() - def delete_book(self, cid): + def delete_node(self, lpath): ''' - Remove DOM node corresponding to book with C{id == cid}. + Remove DOM node corresponding to book with lpath. Also remove book from any collections it is part of. ''' - for book in self: - if str(book.id) == str(cid): - self.remove(book) - self._delete_book(book.elem) - break + for child in self.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + if child.getAttribute('path') == lpath: + self._delete_node(child) + break - def remove_book(self, path): + def remove_book(self, book): ''' Remove DOM node corresponding to book with C{path == path}. - Also remove book from any collections it is part of. + Also remove book from any collections it is part of, and remove + from the booklist ''' - for book in self: - if path.endswith(book.rpath): - self.remove(book) - self._delete_book(book.elem) - break + self.remove(book) + self.delete_node(book.lpath) def playlists(self): ans = [] @@ -358,15 +275,6 @@ class BookList(_BookList): item.setAttribute('id', str(id)) coll.appendChild(item) - def get_playlists(self, bookid): - ans = [] - for pl in self.playlists(): - for item in pl.childNodes: - if hasattr(item, 'tagName') and item.tagName.endswith('item'): - if item.getAttribute('id') == str(bookid): - ans.append(pl) - return ans - def next_id(self): return self.document.documentElement.getAttribute('nextID') @@ -378,27 +286,36 @@ class BookList(_BookList): src = self.document.toxml('utf-8') + '\n' stream.write(src.replace("'", ''')) - def book_by_id(self, id): - for book in self: - if str(book.id) == str(id): - return book - def reorder_playlists(self): for title in self.tag_order.keys(): pl = self.playlist_by_title(title) if not pl: continue - db_ids = [i.getAttribute('id') for i in pl.childNodes if hasattr(i, 'getAttribute')] - pl_book_ids = [getattr(self.book_by_id(i), 'db_id', None) for i in db_ids] + # make a list of the ids + sony_ids = [id.getAttribute('id') \ + for id in pl.childNodes if hasattr(id, 'getAttribute')] + # convert IDs in playlist to a list of lpaths + sony_paths = [self.sony_id_cache[id] for id in sony_ids] + # create list of books containing lpaths + books = [self.books_lpath_cache.get(p, None) for p in sony_paths] + # create dict of db_id -> sony_id imap = {} - for i, j in zip(pl_book_ids, db_ids): - imap[i] = j - pl_book_ids = [i for i in pl_book_ids if i is not None] - ordered_ids = [i for i in self.tag_order[title] if i in pl_book_ids] + for book, sony_id in zip(books, sony_ids): + if book is not None: + db_id = book.application_id + if db_id is None: + db_id = book.db_id + if db_id is not None: + imap[book.application_id] = sony_id + # filter the list, removing books not on device but on playlist + books = [i for i in books if i is not None] + # filter the order specification to the books we have + ordered_ids = [db_id for db_id in self.tag_order[title] if db_id in imap] + # rewrite the playlist in the correct order if len(ordered_ids) < len(pl.childNodes): continue - children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] + children = [i for i in pl.childNodes if hasattr(i, 'getAttribute')] for child in children: pl.removeChild(child) child.unlink() @@ -439,8 +356,12 @@ def fix_ids(main, carda, cardb): except KeyError: item.parentNode.removeChild(item) item.unlink() - db.reorder_playlists() + db.sony_id_cache = {} + for child in db.root_element.childNodes: + if child.nodeType == child.ELEMENT_NODE and child.hasAttribute("id"): + db.sony_id_cache[child.getAttribute('id')] = child.getAttribute('path') + regen_ids(main) regen_ids(carda) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index f4fc4b0d29..d2823ff4a4 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -11,15 +11,14 @@ Device driver for the SONY PRS-505 import os import re -import time -from itertools import cycle -from calibre.devices.usbms.cli import CLI -from calibre.devices.usbms.device import Device -from calibre.devices.prs505.books import BookList, fix_ids +from calibre.devices.usbms.driver import USBMS +from calibre.devices.prs505.books import BookList as PRS_BookList, fix_ids +from calibre.devices.prs505 import MEDIA_XML +from calibre.devices.prs505 import CACHE_XML from calibre import __appname__ -class PRS505(CLI, Device): +class PRS505(USBMS): name = 'PRS-300/505 Device Interface' gui_name = 'SONY Reader' @@ -28,6 +27,8 @@ class PRS505(CLI, Device): supported_platforms = ['windows', 'osx', 'linux'] path_sep = '/' + booklist_class = PRS_BookList # See USBMS for some explanation of this + FORMATS = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt'] VENDOR_ID = [0x054c] #: SONY Vendor Id @@ -46,9 +47,6 @@ class PRS505(CLI, Device): MAIN_MEMORY_VOLUME_LABEL = 'Sony Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'Sony Reader Storage Card' - MEDIA_XML = 'database/cache/media.xml' - CACHE_XML = 'Sony Reader/database/cache.xml' - CARD_PATH_PREFIX = __appname__ SUPPORTS_SUB_DIRS = True @@ -63,64 +61,9 @@ class PRS505(CLI, Device): def windows_filter_pnp_id(self, pnp_id): return '_LAUNCHER' in pnp_id - def open(self): - self.report_progress = lambda x, y: x - Device.open(self) - - def write_cache(prefix): - try: - cachep = os.path.join(prefix, *(self.CACHE_XML.split('/'))) - if not os.path.exists(cachep): - dname = os.path.dirname(cachep) - if not os.path.exists(dname): - try: - os.makedirs(dname, mode=0777) - except: - time.sleep(5) - os.makedirs(dname, mode=0777) - with open(cachep, 'wb') as f: - f.write(u''' - - - '''.encode('utf8')) - return True - except: - import traceback - traceback.print_exc() - return False - - if self._card_a_prefix is not None: - if not write_cache(self._card_a_prefix): - self._card_a_prefix = None - if self._card_b_prefix is not None: - if not write_cache(self._card_b_prefix): - self._card_b_prefix = None - def get_device_information(self, end_session=True): return (self.gui_name, '', '', '') - def books(self, oncard=None, end_session=True): - if oncard == 'carda' and not self._card_a_prefix: - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - elif oncard == 'cardb' and not self._card_b_prefix: - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - elif oncard and oncard != 'carda' and oncard != 'cardb': - self.report_progress(1.0, _('Getting list of books on device...')) - return [] - - db = self.__class__.CACHE_XML if oncard else self.__class__.MEDIA_XML - prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix - bl = BookList(open(prefix + db, 'rb'), prefix, self.report_progress) - paths = bl.purge_corrupted_files() - for path in paths: - path = os.path.join(prefix, path) - if os.path.exists(path): - os.unlink(path) - self.report_progress(1.0, _('Getting list of books on device...')) - return bl - def filename_callback(self, fname, mi): if getattr(mi, 'application_id', None) is not None: base = fname.rpartition('.')[0] @@ -129,90 +72,16 @@ class PRS505(CLI, Device): fname = base + suffix + '.' + fname.rpartition('.')[-1] return fname - def upload_books(self, files, names, on_card=None, end_session=True, - metadata=None): - - path = self._sanity_check(on_card, files) - - paths, ctimes, sizes = [], [], [] - names = iter(names) - metadata = iter(metadata) - for i, infile in enumerate(files): - mdata, fname = metadata.next(), names.next() - filepath = self.create_upload_path(path, mdata, fname) - - paths.append(filepath) - self.put_file(infile, paths[-1], replace_file=True) - ctimes.append(os.path.getctime(paths[-1])) - sizes.append(os.stat(paths[-1]).st_size) - - self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) - - self.report_progress(1.0, _('Transferring books to device...')) - - return zip(paths, sizes, ctimes, cycle([on_card])) - - def add_books_to_metadata(self, locations, metadata, booklists): - if not locations or not metadata: - return - - metadata = iter(metadata) - for location in locations: - info = metadata.next() - path = location[0] - oncard = location[3] - blist = 2 if oncard == 'cardb' else 1 if oncard == 'carda' else 0 - - if self._main_prefix and path.startswith(self._main_prefix): - name = path.replace(self._main_prefix, '') - elif self._card_a_prefix and path.startswith(self._card_a_prefix): - name = path.replace(self._card_a_prefix, '') - elif self._card_b_prefix and path.startswith(self._card_b_prefix): - name = path.replace(self._card_b_prefix, '') - - name = name.replace('\\', '/') - name = name.replace('//', '/') - if name.startswith('/'): - name = name[1:] - - opts = self.settings() - collections = opts.extra_customization.split(',') if opts.extra_customization else [] - booklist = booklists[blist] - if not hasattr(booklist, 'add_book'): - raise ValueError(('Incorrect upload location %s. Did you choose the' - ' correct card A or B, to send books to?')%oncard) - booklist.add_book(info, name, collections, *location[1:-1]) - fix_ids(*booklists) - - def delete_books(self, paths, end_session=True): - for i, path in enumerate(paths): - self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) - if os.path.exists(path): - os.unlink(path) - try: - os.removedirs(os.path.dirname(path)) - except: - pass - self.report_progress(1.0, _('Removing books from device...')) - - @classmethod - def remove_books_from_metadata(cls, paths, booklists): - for path in paths: - for bl in booklists: - if hasattr(bl, 'remove_book'): - bl.remove_book(path) - fix_ids(*booklists) - def sync_booklists(self, booklists, end_session=True): fix_ids(*booklists) if not os.path.exists(self._main_prefix): os.makedirs(self._main_prefix) - with open(self._main_prefix + self.__class__.MEDIA_XML, 'wb') as f: + with open(self._main_prefix + MEDIA_XML, 'wb') as f: booklists[0].write(f) def write_card_prefix(prefix, listid): if prefix is not None and hasattr(booklists[listid], 'write'): - tgt = os.path.join(prefix, *(self.CACHE_XML.split('/'))) + tgt = os.path.join(prefix, *(CACHE_XML.split('/'))) base = os.path.dirname(tgt) if not os.path.exists(base): os.makedirs(base) @@ -221,8 +90,7 @@ class PRS505(CLI, Device): write_card_prefix(self._card_a_prefix, 1) write_card_prefix(self._card_b_prefix, 2) - self.report_progress(1.0, _('Sending metadata to device...')) - + USBMS.sync_booklists(self, booklists, end_session) class PRS700(PRS505): @@ -241,5 +109,3 @@ class PRS700(PRS505): OSX_MAIN_MEM = re.compile(r'Sony PRS-((700/[^:]+)|((6|9)00)) Media') OSX_CARD_A_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))MS Media') OSX_CARD_B_MEM = re.compile(r'Sony PRS-((700/[^:]+:)|((6|9)00 ))SD Media') - - diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index eca9a27096..b153300282 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -31,14 +31,14 @@ class Book(MetaInformation): MetaInformation.__init__(self, '') self.path = os.path.join(prefix, lpath) - self.lpath = lpath + if os.sep == '\\': + self.path = self.path.replace('/', '\\') + self.lpath = lpath.replace('\\', '/') + else: + self.lpath = lpath self.mime = mime_type_ext(path_to_ext(lpath)) - self.size = os.stat(self.path).st_size if size == None else size - self.db_id = None - try: - self.datetime = time.gmtime(os.path.getctime(self.path)) - except ValueError: - self.datetime = time.gmtime() + self.size = None # will be set later + self.datetime = time.gmtime() if other: self.smart_update(other) @@ -66,6 +66,16 @@ class Book(MetaInformation): return spath == opath + @dynamic_property + def db_id(self): + doc = '''The database id in the application database that this file corresponds to''' + def fget(self): + match = re.search(r'_(\d+)$', self.lpath.rpartition('.')[0]) + if match: + return int(match.group(1)) + return None + return property(fget=fget, doc=doc) + @dynamic_property def title_sorter(self): doc = '''String to sort the title. If absent, title is returned''' @@ -77,13 +87,6 @@ class Book(MetaInformation): def thumbnail(self): return None -# def __str__(self): -# ''' -# Return a utf-8 encoded string with title author and path information -# ''' -# return self.title.encode('utf-8') + " by " + \ -# self.authors.encode('utf-8') + " at " + self.path.encode('utf-8') - def smart_update(self, other): ''' Merge the information in C{other} into self. In case of conflicts, the information @@ -105,9 +108,26 @@ class Book(MetaInformation): class BookList(_BookList): + def __init__(self, oncard, prefix, settings): + pass + def supports_tags(self): return True def set_tags(self, book, tags): book.tags = tags + def add_book(self, book, replace_metadata): + ''' + Add the book to the booklist. Intent is to maintain any device-internal + metadata. Return True if booklists must be sync'ed + ''' + if book not in self: + self.append(book) + + def remove_book(self, book): + ''' + Remove a book from the booklist. Correct any device metadata at the + same time + ''' + self.remove(book) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 1b048d1bb6..249733b4e3 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -113,15 +113,17 @@ class Device(DeviceConfig, DevicePlugin): def _windows_space(cls, prefix): if not prefix: return 0, 0 + if prefix.endswith(os.sep): + prefix = prefix[:-1] win32file = __import__('win32file', globals(), locals(), [], -1) try: sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \ - win32file.GetDiskFreeSpace(prefix[:-1]) + win32file.GetDiskFreeSpace(prefix) except Exception, err: if getattr(err, 'args', [None])[0] == 21: # Disk not ready time.sleep(3) sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \ - win32file.GetDiskFreeSpace(prefix[:-1]) + win32file.GetDiskFreeSpace(prefix) else: raise mult = sectors_per_cluster * bytes_per_sector return total_clusters * mult, free_clusters * mult diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 410fc9bef5..332f337a2f 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -27,6 +27,13 @@ class USBMS(CLI, Device): author = _('John Schember') supported_platforms = ['windows', 'osx', 'linux'] + # Store type instances of BookList and Book. We must do this because + # a) we need to override these classes in some device drivers, and + # b) the classmethods seem only to see real attributes declared in the + # class, not attributes stored in the class + booklist_class = BookList + book_class = Book + FORMATS = [] CAN_SET_METADATA = True METADATA_CACHE = 'metadata.calibre' @@ -37,48 +44,52 @@ class USBMS(CLI, Device): def books(self, oncard=None, end_session=True): from calibre.ebooks.metadata.meta import path_to_ext - bl = BookList() - metadata = BookList() - need_sync = False if oncard == 'carda' and not self._card_a_prefix: self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] elif oncard == 'cardb' and not self._card_b_prefix: self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] elif oncard and oncard != 'carda' and oncard != 'cardb': self.report_progress(1.0, _('Getting list of books on device...')) - return bl + return [] + + prefix = self._card_a_prefix if oncard == 'carda' else \ + self._card_b_prefix if oncard == 'cardb' \ + else self._main_prefix - prefix = self._card_a_prefix if oncard == 'carda' else self._card_b_prefix if oncard == 'cardb' else self._main_prefix ebook_dirs = self.EBOOK_DIR_CARD_A if oncard == 'carda' else \ self.EBOOK_DIR_CARD_B if oncard == 'cardb' else \ self.get_main_ebook_dir() + # build a temporary list of books from the metadata cache bl, need_sync = self.parse_metadata_cache(prefix, self.METADATA_CACHE) - # make a dict cache of paths so the lookup in the loop below is faster. bl_cache = {} for idx,b in enumerate(bl): - bl_cache[b.path] = idx + bl_cache[b.lpath] = idx self.count_found_in_bl = 0 + # Make the real booklist that will be filled in below + metadata = self.booklist_class(oncard, prefix, self.settings) + def update_booklist(filename, path, prefix): changed = False if path_to_ext(filename) in self.FORMATS: try: - lpath = os.path.join(path, filename).partition(prefix)[2] + lpath = os.path.join(path, filename).partition(self.normalize_path(prefix))[2] if lpath.startswith(os.sep): lpath = lpath[len(os.sep):] - p = os.path.join(prefix, lpath) - if p in bl_cache: - item, changed = self.__class__.update_metadata_item(bl[bl_cache[p]]) + idx = bl_cache.get(lpath.replace('\\', '/'), None) + if idx is not None: + item, changed = self.update_metadata_item(bl[idx]) self.count_found_in_bl += 1 else: - item = self.__class__.book_from_path(prefix, lpath) + item = self.book_from_path(prefix, lpath) + changed = True + if metadata.add_book(item, replace_metadata=False): changed = True - metadata.append(item) except: # Probably a filename encoding error import traceback traceback.print_exc() @@ -87,7 +98,9 @@ class USBMS(CLI, Device): if isinstance(ebook_dirs, basestring): ebook_dirs = [ebook_dirs] for ebook_dir in ebook_dirs: - ebook_dir = os.path.join(prefix, *(ebook_dir.split('/'))) if ebook_dir else prefix + ebook_dir = self.normalize_path( \ + os.path.join(prefix, *(ebook_dir.split('/'))) \ + if ebook_dir else prefix) if not os.path.exists(ebook_dir): continue # Get all books in the ebook_dir directory if self.SUPPORTS_SUB_DIRS: @@ -108,6 +121,7 @@ class USBMS(CLI, Device): # if count != len(bl) then there were items in it that we did not # find on the device. If need_sync is True then there were either items # on the device that were not in bl or some of the items were changed. + print "count found in cache: %d, count of files in cache: %d, must_sync_cache: %s" % (self.count_found_in_bl, len(bl), need_sync) if self.count_found_in_bl != len(bl) or need_sync: if oncard == 'cardb': self.sync_booklists((None, None, metadata)) @@ -117,12 +131,10 @@ class USBMS(CLI, Device): self.sync_booklists((metadata, None, None)) self.report_progress(1.0, _('Getting list of books on device...')) - #print 'at return', now() - start_time return metadata def upload_books(self, files, names, on_card=None, end_session=True, metadata=None): - path = self._sanity_check(on_card, files) paths = [] @@ -131,13 +143,12 @@ class USBMS(CLI, Device): for i, infile in enumerate(files): mdata, fname = metadata.next(), names.next() - filepath = self.create_upload_path(path, mdata, fname) - + filepath = self.normalize_path(self.create_upload_path(path, mdata, fname)) paths.append(filepath) - - self.put_file(infile, filepath, replace_file=True) + self.put_file(self.normalize_path(infile), filepath, replace_file=True) try: - self.upload_cover(os.path.dirname(filepath), os.path.splitext(os.path.basename(filepath))[0], mdata) + self.upload_cover(os.path.dirname(filepath), + os.path.splitext(os.path.basename(filepath))[0], mdata) except: # Failure to upload cover is not catastrophic import traceback traceback.print_exc() @@ -145,7 +156,6 @@ class USBMS(CLI, Device): self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) self.report_progress(1.0, _('Transferring books to device...')) - return zip(paths, cycle([on_card])) def upload_cover(self, path, filename, metadata): @@ -166,25 +176,32 @@ class USBMS(CLI, Device): blist = 2 if location[1] == 'cardb' else 1 if location[1] == 'carda' else 0 if self._main_prefix: + # Normalize path and prefix + if self._main_prefix.find('\\') >= 0: + path = path.replace('/', '\\') + else: + path = path.replace('\\', '/') prefix = self._main_prefix if path.startswith(self._main_prefix) else None if not prefix and self._card_a_prefix: prefix = self._card_a_prefix if path.startswith(self._card_a_prefix) else None if not prefix and self._card_b_prefix: prefix = self._card_b_prefix if path.startswith(self._card_b_prefix) else None + if prefix is None: + print 'in add_books_to_metadata. Prefix is None!', path, self._main_prefix + continue lpath = path.partition(prefix)[2] - if lpath.startswith(os.sep): - lpath = lpath[len(os.sep):] - - book = Book(prefix, lpath, other=info) - - if book not in booklists[blist]: - booklists[blist].append(book) - + if lpath.startswith('/') or lpath.startswith('\\'): + lpath = lpath[1:] + book = self.book_class(prefix, lpath, other=info) + if book.size is None: + book.size = os.stat(self.normalize_path(path)).st_size + booklists[blist].add_book(book, replace_metadata=True) self.report_progress(1.0, _('Adding books to device metadata listing...')) def delete_books(self, paths, end_session=True): for i, path in enumerate(paths): self.report_progress((i+1) / float(len(paths)), _('Removing books from device...')) + path = self.normalize_path(path) if os.path.exists(path): # Delete the ebook os.unlink(path) @@ -209,19 +226,19 @@ class USBMS(CLI, Device): for bl in booklists: for book in bl: if path.endswith(book.path): - bl.remove(book) + bl.remove_book(book) self.report_progress(1.0, _('Removing books from device metadata listing...')) def sync_booklists(self, booklists, end_session=True): - if not os.path.exists(self._main_prefix): - os.makedirs(self._main_prefix) + if not os.path.exists(self.normalize_path(self._main_prefix)): + os.makedirs(self.normalize_path(self._main_prefix)) def write_prefix(prefix, listid): - if prefix is not None and isinstance(booklists[listid], BookList): + if prefix is not None and isinstance(booklists[listid], self.booklist_class): if not os.path.exists(prefix): - os.makedirs(prefix) + os.makedirs(self.normalize_path(prefix)) js = [item.to_json() for item in booklists[listid]] - with open(os.path.join(prefix, self.METADATA_CACHE), 'wb') as f: + with open(self.normalize_path(os.path.join(prefix, self.METADATA_CACHE)), 'wb') as f: json.dump(js, f, indent=2, encoding='utf-8') write_prefix(self._main_prefix, 0) write_prefix(self._card_a_prefix, 1) @@ -229,37 +246,45 @@ class USBMS(CLI, Device): self.report_progress(1.0, _('Sending metadata to device...')) + @classmethod + def normalize_path(cls, path): + if path is None: + return None + if os.sep == '\\': + path = path.replace('/', '\\') + else: + path = path.replace('\\', '/') + return path + @classmethod def parse_metadata_cache(cls, prefix, name): + bl = [] js = [] - bl = BookList() need_sync = False try: - with open(os.path.join(prefix, name), 'rb') as f: + with open(cls.normalize_path(os.path.join(prefix, name)), 'rb') as f: js = json.load(f, encoding='utf-8') for item in js: - lpath = item.get('lpath', None) - if not lpath or not os.path.exists(os.path.join(prefix, lpath)): - need_sync = True - continue - book = Book(prefix, lpath) + book = cls.book_class(prefix, item.get('lpath', None)) for key in item.keys(): setattr(book, key, item[key]) bl.append(book) except: import traceback traceback.print_exc() - bl = BookList() + bl = [] + need_sync = True return bl, need_sync @classmethod def update_metadata_item(cls, item): changed = False - size = os.stat(item.path).st_size + size = os.stat(cls.normalize_path(item.path)).st_size if size != item.size: changed = True mi = cls.metadata_from_path(item.path) item.smart_update(mi) + item.size = size return item, changed @classmethod @@ -278,15 +303,15 @@ class USBMS(CLI, Device): from calibre.ebooks.metadata import MetaInformation if cls.settings().read_metadata or cls.MUST_READ_METADATA: - mi = cls.metadata_from_path(os.path.join(prefix, path)) + mi = cls.metadata_from_path(cls.normalize_path(os.path.join(prefix, path))) else: from calibre.ebooks.metadata.meta import metadata_from_filename - mi = metadata_from_filename(os.path.basename(path), + mi = metadata_from_filename(cls.normalize_path(os.path.basename(path)), re.compile(r'^(?P[ \S]+?)[ _]-[ _](?P<author>[ \S]+?)_+\d+')) if mi is None: mi = MetaInformation(os.path.splitext(os.path.basename(path))[0], [_('Unknown')]) - - book = Book(prefix, path, other=mi) + mi.size = os.stat(cls.normalize_path(os.path.join(prefix, path))).st_size + book = cls.book_class(prefix, path, other=mi) return book diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 60dffc0cf7..a1c29be337 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -254,11 +254,11 @@ class MetaInformation(object): setattr(self, x, getattr(mi, x, None)) def print_all_attributes(self): - for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher', - 'series', 'series_index', 'rating', 'isbn', 'language', + for x in ('author', 'author_sort', 'title_sort', 'comments', 'category', 'publisher', + 'series', 'series_index', 'tags', 'rating', 'isbn', 'language', 'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', - 'rights', 'publication_type', 'uuid', + 'rights', 'publication_type', 'uuid', 'tag_order', ): prints(x, getattr(self, x, 'None')) @@ -278,7 +278,7 @@ class MetaInformation(object): 'isbn', 'application_id', 'manifest', 'spine', 'toc', 'cover', 'language', 'guide', 'book_producer', 'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights', - 'publication_type', 'uuid',): + 'publication_type', 'uuid', 'tag_order'): if hasattr(mi, attr): val = getattr(mi, attr) if val is not None: diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index f890515aa5..31fe4bbbbd 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -25,6 +25,7 @@ from calibre.utils.filenames import ascii_filename from calibre.devices.errors import FreeSpaceError from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config +from calibre.devices.folder_device.driver import FOLDER_DEVICE class DeviceJob(BaseJob): @@ -207,6 +208,23 @@ class DeviceManager(Thread): return self.create_job(self._get_device_information, done, description=_('Get device information')) + def connect_to_folder(self, path): + dev = FOLDER_DEVICE(path) + try: + dev.open() + except: + print 'Unable to open device', dev + traceback.print_exc() + return False + self.connected_device = dev + self.connected_slot(True) + return True + + def disconnect_folder(self): + if self.connected_device is not None: + if hasattr(self.connected_device, 'disconnect_from_folder'): + self.connected_device.disconnect_from_folder() + def _books(self): '''Get metadata from device''' mainlist = self.device.books(oncard=None, end_session=False) @@ -309,6 +327,8 @@ class DeviceAction(QAction): class DeviceMenu(QMenu): fetch_annotations = pyqtSignal() + connect_to_folder = pyqtSignal() + disconnect_from_folder = pyqtSignal() def __init__(self, parent=None): QMenu.__init__(self, parent) @@ -404,6 +424,18 @@ class DeviceMenu(QMenu): if opts.accounts: self.addSeparator() self.addMenu(self.email_to_menu) + + self.addSeparator() + mitem = self.addAction(_('Connect to folder')) + mitem.setEnabled(True) + mitem.triggered.connect(lambda x : self.connect_to_folder.emit()) + self.connect_to_folder_action = mitem + + mitem = self.addAction(_('Disconnect from folder')) + mitem.setEnabled(False) + mitem.triggered.connect(lambda x : self.disconnect_from_folder.emit()) + self.disconnect_from_folder_action = mitem + self.addSeparator() annot = self.addAction(_('Fetch annotations (experimental)')) annot.setEnabled(False) @@ -523,7 +555,8 @@ class DeviceGUI(object): d = ChooseFormatDialog(self, _('Choose format to send to device'), self.device_manager.device.settings().format_map) d.exec_() - fmt = d.format().lower() + if d.format(): + fmt = d.format().lower() dest, sub_dest = dest.split(':') if dest in ('main', 'carda', 'cardb'): if not self.device_connected or not self.device_manager: @@ -821,7 +854,9 @@ class DeviceGUI(object): def sync_to_device(self, on_card, delete_from_library, specific_format=None, send_ids=None, do_auto_convert=True): - ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids + ids = [self.library_view.model().id(r) \ + for r in self.library_view.selectionModel().selectedRows()] \ + if send_ids is None else send_ids if not self.device_manager or not ids or len(ids) == 0: return @@ -842,8 +877,7 @@ class DeviceGUI(object): ids = iter(ids) for mi in metadata: if mi.cover and os.access(mi.cover, os.R_OK): - mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, - 'rb').read()) + mi.thumbnail = self.cover_to_thumbnail(open(mi.cover, 'rb').read()) imetadata = iter(metadata) files = [getattr(f, 'name', None) for f in _files] @@ -890,7 +924,9 @@ class DeviceGUI(object): bad.append(self.library_view.model().db.title(id, index_is_id=True)) if auto != []: - format = specific_format if specific_format in list(set(settings.format_map).intersection(set(available_output_formats()))) else None + format = specific_format if specific_format in \ + list(set(settings.format_map).intersection(set(available_output_formats()))) \ + else None if not format: for fmt in settings.format_map: if fmt in list(set(settings.format_map).intersection(set(available_output_formats()))): @@ -995,83 +1031,111 @@ class DeviceGUI(object): if changed: self.library_view.model().refresh_ids(list(changed)) - def book_on_device(self, index, format=None, reset=False): + def book_on_device(self, id, format=None, reset=False): loc = [None, None, None] if reset: - self.book_on_device_cache = None + self.book_db_title_cache = None + self.book_db_uuid_cache = None return - if self.book_on_device_cache is None: - self.book_on_device_cache = [] + if self.book_db_title_cache is None: + self.book_db_title_cache = [] + self.book_db_uuid_cache = [] for i, l in enumerate(self.booklists()): - self.book_on_device_cache.append({}) + self.book_db_title_cache.append({}) + self.book_db_uuid_cache.append(set()) for book in l: book_title = book.title.lower() if book.title else '' book_title = re.sub('(?u)\W|[_]', '', book_title) - if book_title not in self.book_on_device_cache[i]: - self.book_on_device_cache[i][book_title] = \ + if book_title not in self.book_db_title_cache[i]: + self.book_db_title_cache[i][book_title] = \ {'authors':set(), 'db_ids':set(), 'uuids':set()} book_authors = authors_to_string(book.authors).lower() book_authors = re.sub('(?u)\W|[_]', '', book_authors) - self.book_on_device_cache[i][book_title]['authors'].add(book_authors) - id = getattr(book, 'application_id', None) - if id is None: - id = book.db_id - if id is not None: - self.book_on_device_cache[i][book_title]['db_ids'].add(id) + self.book_db_title_cache[i][book_title]['authors'].add(book_authors) + db_id = getattr(book, 'application_id', None) + if db_id is None: + db_id = book.db_id + if db_id is not None: + self.book_db_title_cache[i][book_title]['db_ids'].add(db_id) uuid = getattr(book, 'uuid', None) - if uuid is None: - self.book_on_device_cache[i][book_title]['uuids'].add(uuid) + if uuid is not None: + self.book_db_uuid_cache[i].add(uuid) - db = self.library_view.model().db - db_title = db.title(index, index_is_id=True).lower() - db_title = re.sub('(?u)\W|[_]', '', db_title) - db_authors = db.authors(index, index_is_id=True) - db_authors = db_authors.lower() if db_authors else '' - db_authors = re.sub('(?u)\W|[_]', '', db_authors) - db_uuid = db.uuid(index, index_is_id=True) + mi = self.library_view.model().db.get_metadata(id, index_is_id=True) for i, l in enumerate(self.booklists()): - d = self.book_on_device_cache[i].get(db_title, None) - if d: - if db_uuid in d['uuids'] or \ - index in d['db_ids'] or \ - db_authors in d['authors']: + if mi.uuid in self.book_db_uuid_cache[i]: + loc[i] = True + continue + db_title = re.sub('(?u)\W|[_]', '', mi.title.lower()) + cache = self.book_db_title_cache[i].get(db_title, None) + if cache: + if id in cache['db_ids']: + loc[i] = True + break + if mi.authors and \ + re.sub('(?u)\W|[_]', '', authors_to_string(mi.authors).lower()) \ + in cache['authors']: loc[i] = True break return loc def set_books_in_library(self, booklists, reset=False): if reset: - # First build a self.book_in_library_cache of the library, so the search isn't On**2 - self.book_in_library_cache = {} - for id, title in self.library_view.model().db.all_titles(): - title = re.sub('(?u)\W|[_]', '', title.lower()) - if title not in self.book_in_library_cache: - self.book_in_library_cache[title] = {'authors':set(), 'db_ids':set(), 'uuids':set()} - au = self.library_view.model().db.authors(id, index_is_id=True) - authors = au.lower() if au else '' + # First build a cache of the library, so the search isn't On**2 + self.db_book_title_cache = {} + self.db_book_uuid_cache = set() + db = self.library_view.model().db + for id in db.data.iterallids(): + mi = db.get_metadata(id, index_is_id=True) + title = re.sub('(?u)\W|[_]', '', mi.title.lower()) + if title not in self.db_book_title_cache: + self.db_book_title_cache[title] = {'authors':{}, 'db_ids':{}} + authors = authors_to_string(mi.authors).lower() if mi.authors else '' authors = re.sub('(?u)\W|[_]', '', authors) - self.book_in_library_cache[title]['authors'].add(authors) - self.book_in_library_cache[title]['db_ids'].add(id) - self.book_in_library_cache[title]['uuids'].add(self.library_view.model().db.uuid(id, index_is_id=True)) + self.db_book_title_cache[title]['authors'][authors] = mi + self.db_book_title_cache[title]['db_ids'][mi.application_id] = mi + self.db_book_uuid_cache.add(mi.uuid) - # Now iterate through all the books on the device, setting the in_library field + # Now iterate through all the books on the device, setting the + # in_library field Fastest and most accurate key is the uuid. Second is + # the application_id, which is really the db key, but as this can + # accidentally match across libraries we also verify the title. The + # db_id exists on Sony devices. Fallback is title and author match + resend_metadata = False for booklist in booklists: for book in booklist: + if getattr(book, 'uuid', None) in self.db_book_uuid_cache: + book.in_library = True + continue + book_title = book.title.lower() if book.title else '' book_title = re.sub('(?u)\W|[_]', '', book_title) book.in_library = False - d = self.book_in_library_cache.get(book_title, None) + d = self.db_book_title_cache.get(book_title, None) if d is not None: - if getattr(book, 'uuid', None) in d['uuids'] or \ - getattr(book, 'application_id', None) in d['db_ids']: + if getattr(book, 'application_id', None) in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.application_id]) + resend_metadata = True continue if book.db_id in d['db_ids']: book.in_library = True + book.smart_update(d['db_ids'][book.db_id]) + resend_metadata = True continue book_authors = authors_to_string(book.authors).lower() if book.authors else '' book_authors = re.sub('(?u)\W|[_]', '', book_authors) if book_authors in d['authors']: book.in_library = True + book.smart_update(d['authors'][book_authors]) + resend_metadata = True + # Set author_sort if it isn't already + asort = getattr(book, 'author_sort', None) + if not asort: + pass + if resend_metadata: + # Correcting metadata cache on device. + if self.device_manager.is_device_connected: + self.device_manager.sync_booklists(None, booklists) diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index cdebf65489..9a9ffb5d94 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -17,7 +17,7 @@ from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, pyqtSignal, \ SIGNAL, QObject, QSize, QModelIndex, QDate from calibre import strftime -from calibre.ebooks.metadata import fmt_sidx, authors_to_string +from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.gui2 import NONE, TableView, config, error_dialog, UNDEFINED_QDATE from calibre.gui2.dialogs.comments_dialog import CommentsDialog @@ -371,7 +371,7 @@ class BooksModel(QAbstractTableModel): def set_device_connected(self, is_connected): self.device_connected = is_connected self.read_config() - self.refresh(reset=True) + self.db.refresh_ondevice() self.database_changed.emit(self.db) def set_book_on_device_func(self, func): @@ -1378,7 +1378,17 @@ class DeviceBooksModel(BooksModel): def libcmp(x, y): x, y = self.db[x].in_library, self.db[y].in_library return cmp(x, y) - fcmp = strcmp('title_sorter') if col == 0 else strcmp('authors') if col == 1 else \ + def authorcmp(x, y): + ax = getattr(self.db[x], 'author_sort', None) + ay = getattr(self.db[y], 'author_sort', None) + if ax and ay: + x = ax + y = ay + else: + x, y = authors_to_string(self.db[x].authors), \ + authors_to_string(self.db[y].authors) + return cmp(x, y) + fcmp = strcmp('title_sorter') if col == 0 else authorcmp if col == 1 else \ sizecmp if col == 2 else datecmp if col == 3 else tagscmp if col == 4 else libcmp self.map.sort(cmp=fcmp, reverse=descending) if len(self.map) == len(self.db): @@ -1446,9 +1456,9 @@ class DeviceBooksModel(BooksModel): au = self.db[self.map[row]].authors if not au: au = self.unknown - if role == Qt.EditRole: - return QVariant(authors_to_string(au)) - return QVariant(" & ".join(au)) +# if role == Qt.EditRole: +# return QVariant(au) + return QVariant(authors_to_string(au)) elif col == 2: size = self.db[self.map[row]].size return QVariant(BooksView.human_readable(size)) @@ -1501,7 +1511,7 @@ class DeviceBooksModel(BooksModel): self.db[idx].title = val self.db[idx].title_sorter = val elif col == 1: - self.db[idx].authors = val + self.db[idx].authors = string_to_authors(val) elif col == 4: tags = [i.strip() for i in val.split(',')] tags = [t for t in tags if t] diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index ae3450c096..c1e208625b 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -669,6 +669,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): MainWindow.resizeEvent(self, ev) self.search.setMaximumWidth(self.width()-150) + def connect_to_folder(self): + dir = choose_dir(self, 'Select Device Folder', 'Select folder to open') + if dir is not None: + self.device_manager.connect_to_folder(dir) + self._sync_menu.disconnect_from_folder_action.setEnabled(True) + + def disconnect_from_folder(self): + self.device_manager.disconnect_folder() + def _sync_action_triggered(self, *args): m = getattr(self, '_sync_menu', None) if m is not None: @@ -681,6 +690,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), self.dispatch_sync_event) self._sync_menu.fetch_annotations.connect(self.fetch_annotations) + self._sync_menu.connect_to_folder.connect(self.connect_to_folder) + self._sync_menu.disconnect_from_folder.connect(self.disconnect_from_folder) def add_spare_server(self, *args): self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0))) @@ -939,6 +950,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): Called when a device is connected to the computer. ''' if connected: + self._sync_menu.connect_to_folder_action.setEnabled(False) self.device_manager.get_device_information(\ Dispatcher(self.info_read)) self.set_default_thumbnail(\ @@ -952,8 +964,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): self.device_manager.device) self.location_view.model().device_connected(self.device_manager.device) self.eject_action.setEnabled(True) - self.refresh_ondevice_info (device_connected = True) + self.refresh_ondevice_info (device_connected = True, reset_only = True) else: + self._sync_menu.connect_to_folder_action.setEnabled(True) + self._sync_menu.disconnect_from_folder_action.setEnabled(False) self.save_device_view_settings() self.device_connected = False self._sync_menu.enable_device_actions(False) @@ -1022,10 +1036,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): ############################################################################ ### Force the library view to refresh, taking into consideration books information - def refresh_ondevice_info(self, device_connected): - # Save current column widths because we might be turning on OnDevice - self.library_view.write_settings() + def refresh_ondevice_info(self, device_connected, reset_only = False): self.book_on_device(None, reset=True) + if reset_only: + return + self.library_view.write_settings() self.library_view.model().set_device_connected(device_connected) ############################################################################ @@ -1508,6 +1523,11 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): sm = view.selectionModel() sm.select(ci, sm.Select) else: + if not confirm('<p>'+_('The selected books will be ' + '<b>permanently deleted</b> ' + 'from your device. Are you sure?') + +'</p>', 'library_delete_books', self): + return if self.stack.currentIndex() == 1: view = self.memory_view elif self.stack.currentIndex() == 2: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 68ed4cc092..e280a2178b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -557,6 +557,12 @@ class ResultCache(SearchQueryParser): def count(self): return len(self._map) + def refresh_ondevice(self, db): + ondevice_col = self.FIELD_MAP['ondevice'] + for item in self._data: + if item is not None: + item[ondevice_col] = db.book_on_device_string(item[0]) + def refresh(self, db, field=None, ascending=True): temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index fd59503eed..5971333078 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -245,6 +245,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.has_id = self.data.has_id self.count = self.data.count + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) + self.refresh() self.last_update_check = self.last_modified() @@ -470,14 +472,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): im = PILImage.open(f) im.convert('RGB').save(path, 'JPEG') - def book_on_device(self, index): + def book_on_device(self, id): if callable(self.book_on_device_func): - return self.book_on_device_func(index) + return self.book_on_device_func(id) return None - def book_on_device_string(self, index): + def book_on_device_string(self, id): loc = [] - on = self.book_on_device(index) + on = self.book_on_device(id) if on is not None: m, a, b = on if m is not None: