From b7f2788244479b03753718d0b4a20799f2a7be6f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 15:11:08 +0530 Subject: [PATCH 01/39] ... --- src/calibre/ebooks/pdf/writer.py | 14 +++++++++----- src/calibre/gui2/convert/__init__.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index d6ffa42107..d37db69a81 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -137,11 +137,15 @@ class Page(QWebPage): std = {'serif':opts.pdf_serif_family, 'sans':opts.pdf_sans_family, 'mono':opts.pdf_mono_family}.get(opts.pdf_standard_font, opts.pdf_serif_family) - settings.setFontFamily(QWebSettings.StandardFont, std) - settings.setFontFamily(QWebSettings.SerifFont, opts.pdf_serif_family) - settings.setFontFamily(QWebSettings.SansSerifFont, - opts.pdf_sans_family) - settings.setFontFamily(QWebSettings.FixedFont, opts.pdf_mono_family) + if std: + settings.setFontFamily(QWebSettings.StandardFont, std) + if opts.pdf_serif_family: + settings.setFontFamily(QWebSettings.SerifFont, opts.pdf_serif_family) + if opts.pdf_sans_family: + settings.setFontFamily(QWebSettings.SansSerifFont, + opts.pdf_sans_family) + if opts.pdf_mono_family: + settings.setFontFamily(QWebSettings.FixedFont, opts.pdf_mono_family) def javaScriptConsoleMessage(self, msg, lineno, msgid): self.log.debug(u'JS:', unicode(msg)) diff --git a/src/calibre/gui2/convert/__init__.py b/src/calibre/gui2/convert/__init__.py index e01238a2e5..38fb641987 100644 --- a/src/calibre/gui2/convert/__init__.py +++ b/src/calibre/gui2/convert/__init__.py @@ -143,7 +143,7 @@ class Widget(QWidget): ans = None return ans elif isinstance(g, QFontComboBox): - ans = unicode(QFontInfo(g.currentFont().family())) + return unicode(QFontInfo(g.currentFont()).family()) elif isinstance(g, EncodingComboBox): ans = unicode(g.currentText()).strip() try: From 7a3d5937e7b650cfa96f62a27b5cf05e20e72cd5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 17:53:45 +0530 Subject: [PATCH 02/39] Fix #1044345 (Coby Kyros 7035 Not recognized) --- src/calibre/devices/android/driver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 45672fdbd1..9e8aa5fe17 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -197,7 +197,8 @@ class ANDROID(USBMS): 'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON', 'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP', 'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD', - 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0'] + 'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0', + 'COBY_MID'] WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE', '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897', 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', @@ -216,7 +217,7 @@ class ANDROID(USBMS): 'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX', 'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE', 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', - 'S5830I_CARD', 'MID7042', 'LINK-CREATE'] + 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -226,7 +227,7 @@ class ANDROID(USBMS): 'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC', 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', - 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042'] + 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035'] OSX_MAIN_MEM = 'Android Device Main Memory' From 944fb4a7febbe46b205c9959950b9d36b9ceb46c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 18:11:14 +0530 Subject: [PATCH 03/39] ... --- src/calibre/utils/filenames.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d9fd12d466..65451dab9c 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -229,6 +229,10 @@ def samefile(src, dst): symlinks, case insensitivity, mapped drives, etc. Returns True iff both paths exist and point to the same file on disk. + + Note: On windows will return True if the two string are identical (upto + case) even if the file does not exist. This is because I have no way of + knowing how reliable the GetFileInformationByHandle method is. ''' if iswindows: return samefile_windows(src, dst) From 359813d7ebce8ad98a04de33fd8d215e7dd483a6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 18:45:27 +0530 Subject: [PATCH 04/39] MTP: Implement getting list of books and their metadata from device --- src/calibre/devices/mtp/base.py | 33 ++++++- src/calibre/devices/mtp/books.py | 38 ++++++++ src/calibre/devices/mtp/driver.py | 101 +++++++++++++++++++- src/calibre/devices/mtp/filesystem_cache.py | 14 +++ src/calibre/devices/mtp/unix/driver.py | 2 +- src/calibre/devices/mtp/windows/driver.py | 2 +- 6 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 src/calibre/devices/mtp/books.py diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 4f8bbc991f..3b71e40619 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -7,10 +7,17 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from functools import wraps +import re +from functools import wraps, partial +from calibre import prints +from calibre.constants import DEBUG from calibre.devices.interface import DevicePlugin +def debug(*args, **kwargs): + if DEBUG: + prints('MTP:', *args, **kwargs) + def synchronous(func): @wraps(func) def synchronizer(self, *args, **kwargs): @@ -53,4 +60,28 @@ class MTPDeviceBase(DevicePlugin): # return False return False + def build_template_regexp(self): + return None + # TODO: Implement this + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + template = None + try: + template = self.save_template().rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + + diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py new file mode 100644 index 0000000000..c02923702e --- /dev/null +++ b/src/calibre/devices/mtp/books.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre.devices.interface import BookList as BL +from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.metadata.book.json_codec import JsonCodec + +class BookList(BL): + + def __init__(self, storage_id): + self.storage_id = storage_id + + def supports_collections(self): + return False + +class Book(Metadata): + + def __init__(self, storage_id, lpath, other=None): + Metadata.__init__(self, _('Unknown'), other=other) + self.storage_id, self.lpath = storage_id, lpath + self.lpath = self.lpath.replace(os.sep, '/') + self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')]) + + def matches_file(self, mtp_file): + return (self.storage_id == mtp_file.storage_id and + self.mtp_relpath == mtp_file.mtp_relpath) + +class JSONCodec(JsonCodec): + pass + diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index c3e34a2be5..f1e9bdbcff 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,10 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, pprint +import json, pprint, traceback from io import BytesIO +from calibre import prints from calibre.constants import iswindows, numeric_version +from calibre.devices.mtp.base import debug +from calibre.ptempfile import SpooledTemporaryFile from calibre.utils.config import from_json, to_json from calibre.utils.date import now, isoformat @@ -25,6 +28,7 @@ class MTP_DEVICE(BASE): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' + CAN_SET_METADATA = [] def _update_drive_info(self, storage, location_code, name=None): import uuid @@ -81,6 +85,98 @@ class MTP_DEVICE(BASE): self._update_drive_info(self.filesystem_cache.storage(sid), location_code, name=name) + def books(self, oncard=None, end_session=True): + from calibre.devices.mtp.books import JSONCodec + from calibre.devices.mtp.books import BookList, Book + sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard, + self._main_id) + if sid is None: + return BookList(None) + + bl = BookList(sid) + # If True then there is a mismatch between the ebooks on the device and + # the metadata cache + need_sync = False + all_books = list(self.filesystem_cache.iterebooks(sid)) + steps = len(all_books) + 2 + count = 0 + + self.report_progress(0, _('Reading metadata from device')) + # Read the cache if it exists + storage = self.filesystem_cache.storage(sid) + cache = storage.find_path((self.METADATA_CACHE,)) + if cache is not None: + json_codec = JSONCodec() + try: + stream = self.get_file(cache) + json_codec.decode_from_file(stream, bl, Book, sid) + except: + need_sync = True + + relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)} + + for mtp_file in all_books: + count += 1 + relpath = mtp_file.mtp_relpath + idx = relpath_cache.get(relpath, None) + if idx is not None: + cached_metadata = bl[idx] + del relpath_cache[relpath] + if cached_metadata.size == mtp_file.size: + debug('Using cached metadata for', + '/'.join(mtp_file.full_path)) + continue # No need to update metadata + book = cached_metadata + else: + book = Book(sid, '/'.join(relpath)) + bl.append(book) + + need_sync = True + self.report_progress(count/steps, _('Reading metadata from %s')% + ('/'.join(relpath))) + try: + book.smart_update(self.read_file_metadata(mtp_file)) + debug('Read metadata for', '/'.join(mtp_file.full_path)) + except: + prints('Failed to read metadata from', + '/'.join(mtp_file.full_path)) + traceback.print_exc() + book.size = mtp_file.size + + # Remove books in the cache that no longer exist + for idx in sorted(relpath_cache.itervalues(), reverse=True): + del bl[idx] + need_sync = True + + if need_sync: + self.report_progress(count/steps, _('Updating metadata cache on device')) + self.write_metadata_cache(storage, bl) + self.report_progress(1, _('Finished reading metadata from device')) + + def read_file_metadata(self, mtp_file): + from calibre.ebooks.metadata.meta import get_metadata + from calibre.customize.ui import quick_metadata + ext = mtp_file.name.rpartition('.')[-1].lower() + stream = self.get_file(mtp_file) + with quick_metadata: + return get_metadata(stream, stream_type=ext, + force_read_metadata=True, + pattern=self.build_template_regexp()) + + def write_metadata_cache(self, storage, bl): + from calibre.devices.mtp.books import JSONCodec + + if bl.storage_id != storage.storage_id: + # Just a sanity check, should never happen + return + + json_codec = JSONCodec() + stream = SpooledTemporaryFile(10*(1024**2)) + json_codec.encode_to_file(stream, bl) + size = stream.tell() + stream.seek(0) + self.put_file(storage, self.METADATA_CACHE, stream, size) + if __name__ == '__main__': dev = MTP_DEVICE(None) dev.startup() @@ -92,8 +188,9 @@ if __name__ == '__main__': cd = dev.detect_managed_devices(devs) if cd is None: raise ValueError('Failed to detect MTP device') + dev.set_progress_reporter(prints) dev.open(cd, None) - pprint.pprint(dev.get_device_information()) + dev.books() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index cd97c5c2ed..ba2206d191 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -14,6 +14,9 @@ from future_builtins import map from calibre import human_readable, prints, force_unicode from calibre.utils.icu import sort_key, lower +from calibre.ebooks import BOOK_EXTENSIONS + +bexts = frozenset(BOOK_EXTENSIONS) class FileOrFolder(object): @@ -50,6 +53,9 @@ class FileOrFolder(object): if self.storage_id == self.object_id: self.storage_prefix = 'mtp:::%s:::'%self.persistent_id + self.is_ebook = (not self.is_folder and + self.name.rpartition('.')[-1].lower() in bexts) + def __repr__(self): name = 'Folder' if self.is_folder else 'File' try: @@ -147,6 +153,9 @@ class FileOrFolder(object): parent = c return parent + @property + def mtp_relpath(self): + return tuple(x.lower() for x in self.full_path[1:]) class FilesystemCache(object): @@ -192,4 +201,9 @@ class FilesystemCache(object): if e.storage_id == storage_id: return e + def iterebooks(self, storage_id): + for x in self.id_map.itervalues(): + if x.storage_id == storage_id and x.is_ebook: + yield x + diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 5d7d767a9b..9244ac198c 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -17,7 +17,6 @@ from calibre.constants import plugins from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous -from calibre.devices.mtp.filesystem_cache import FilesystemCache MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' 'bcd serial manufacturer product') @@ -175,6 +174,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache with self.lock: storage, all_items, all_errs = [], [], [] for sid, capacity in zip([self._main_id, self._carda_id, diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 0506f63054..191d69560d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -17,7 +17,6 @@ from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase -from calibre.devices.mtp.filesystem_cache import FilesystemCache class ThreadingViolation(Exception): @@ -143,6 +142,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache ts = self.total_space() all_storage = [] items = [] From f59ac23c9c3277760be4a9889be59840a4ecaba6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 18:47:02 +0530 Subject: [PATCH 05/39] ... --- src/calibre/devices/mtp/driver.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index f1e9bdbcff..7ba02e2aaa 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -30,6 +30,11 @@ class MTP_DEVICE(BASE): DRIVEINFO = 'driveinfo.calibre' CAN_SET_METADATA = [] + def open(self, devices, library_uuid): + self.current_library_uuid = library_uuid + BASE.open(self, devices, library_uuid) + + # Device information {{{ def _update_drive_info(self, storage, location_code, name=None): import uuid f = storage.find_path((self.DRIVEINFO,)) @@ -55,10 +60,6 @@ class MTP_DEVICE(BASE): self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw)) self.driveinfo = dinfo - def open(self, devices, library_uuid): - self.current_library_uuid = library_uuid - BASE.open(self, devices, library_uuid) - def get_device_information(self, end_session=True): self.report_progress(1.0, _('Get device information...')) self.driveinfo = {} @@ -84,7 +85,9 @@ class MTP_DEVICE(BASE): return self._update_drive_info(self.filesystem_cache.storage(sid), location_code, name=name) + # }}} + # Get list of books from device, with metadata {{{ def books(self, oncard=None, end_session=True): from calibre.devices.mtp.books import JSONCodec from calibre.devices.mtp.books import BookList, Book @@ -176,6 +179,7 @@ class MTP_DEVICE(BASE): size = stream.tell() stream.seek(0) self.put_file(storage, self.METADATA_CACHE, stream, size) + # }}} if __name__ == '__main__': dev = MTP_DEVICE(None) From 2292991006ae91ec1678f7e0d9edb07727840bc4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 19:40:09 +0530 Subject: [PATCH 06/39] EPUB metadata: When there are multiple tags use the one with the earliest date as the published date --- src/calibre/ebooks/metadata/opf2.py | 41 +++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index d132fe15d0..7f8a01f8bd 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -510,6 +510,7 @@ class OPF(object): # {{{ tags_path = XPath('descendant::*[re:match(name(), "subject", "i")]') isbn_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+ '(re:match(@scheme, "isbn", "i") or re:match(@opf:scheme, "isbn", "i"))]') + pubdate_path = XPath('descendant::*[re:match(name(), "date", "i")]') raster_cover_path = XPath('descendant::*[re:match(name(), "meta", "i") and ' + 're:match(@name, "cover", "i") and @content]') identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]') @@ -538,8 +539,6 @@ class OPF(object): # {{{ formatter=float, none_is=1) title_sort = TitleSortField('title_sort', is_dc=False) rating = MetadataField('rating', is_dc=False, formatter=float) - pubdate = MetadataField('date', formatter=parse_date, - renderer=isoformat) publication_type = MetadataField('publication_type', is_dc=False) timestamp = MetadataField('timestamp', is_dc=False, formatter=parse_date, renderer=isoformat) @@ -852,6 +851,44 @@ class OPF(object): # {{{ return property(fget=fget, fset=fset) + @dynamic_property + def pubdate(self): + + def fget(self): + ans = None + for match in self.pubdate_path(self.metadata): + try: + val = parse_date(etree.tostring(match, encoding=unicode, + method='text', with_tail=False).strip()) + except: + continue + if ans is None or val < ans: + ans = val + return ans + + def fset(self, val): + least_val = least_elem = None + for match in self.pubdate_path(self.metadata): + try: + cval = parse_date(etree.tostring(match, encoding=unicode, + method='text', with_tail=False).strip()) + except: + match.getparent().remove(match) + else: + if not val: + match.getparent().remove(match) + if least_val is None or cval < least_val: + least_val, least_elem = cval, match + + if val: + if least_val is None: + least_elem = self.create_metadata_element('date') + + least_elem.attrib.clear() + least_elem.text = isoformat(val) + + return property(fget=fget, fset=fset) + @dynamic_property def isbn(self): From 8bec5211c19513b8a55ecaad62a1c0177d36cea7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 21:50:23 +0530 Subject: [PATCH 07/39] Refactor to make build_template_regexp a utility function --- src/calibre/devices/__init__.py | 25 +++++++++++++++++++++- src/calibre/devices/mtp/base.py | 32 ++++++++--------------------- src/calibre/devices/usbms/driver.py | 23 +++------------------ 3 files changed, 35 insertions(+), 45 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 37ec55c149..ab772c6905 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint, operator +import sys, time, pprint, operator, re from functools import partial from StringIO import StringIO @@ -27,6 +27,29 @@ def strftime(epoch, zone=time.gmtime): src[2] = INVERSE_MONTH_MAP[int(src[2])] return ' '.join(src) +def build_template_regexp(template): + from calibre import prints + + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + + try: + template = template.rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 3b71e40619..346a44e6e8 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -7,8 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re -from functools import wraps, partial +from functools import wraps from calibre import prints from calibre.constants import DEBUG @@ -61,27 +60,12 @@ class MTPDeviceBase(DevicePlugin): return False def build_template_regexp(self): - return None - # TODO: Implement this - def replfunc(match, seen=None): - v = match.group(1) - if v in ['authors', 'author_sort']: - v = 'author' - if v in ('title', 'series', 'series_index', 'isbn', 'author'): - if v not in seen: - seen.add(v) - return '(?P<' + v + '>.+?)' - return '(.+?)' - s = set() - f = partial(replfunc, seen=s) - template = None - try: - template = self.save_template().rpartition('/')[2] - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - except: - prints(u'Failed to parse template: %r'%template) - template = u'{title} - {authors}' - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - + from calibre.devices import build_template_regexp + # TODO: Use the device specific template here + return build_template_regexp(self.default_save_template) + @property + def default_save_template(cls): + from calibre.library.save_to_disk import config + return config().parse().send_template diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 12e30073ac..f6c7556fd8 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import os, re, time, json, functools, shutil +import os, time, json, shutil from itertools import cycle from calibre.constants import numeric_version @@ -404,25 +404,8 @@ class USBMS(CLI, Device): @classmethod def build_template_regexp(cls): - def replfunc(match, seen=None): - v = match.group(1) - if v in ['authors', 'author_sort']: - v = 'author' - if v in ('title', 'series', 'series_index', 'isbn', 'author'): - if v not in seen: - seen.add(v) - return '(?P<' + v + '>.+?)' - return '(.+?)' - s = set() - f = functools.partial(replfunc, seen=s) - template = None - try: - template = cls.save_template().rpartition('/')[2] - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') - except: - prints(u'Failed to parse template: %r'%template) - template = u'{title} - {authors}' - return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + from calibre.devices import build_template_regexp + return build_template_regexp(cls.save_template()) @classmethod def path_to_unicode(cls, path): From 0c4227c036b285f9dd71da74047b7154f5968785 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 22:49:20 +0530 Subject: [PATCH 08/39] ... --- src/calibre/utils/magick/magick.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index 8713d2cebb..f01af3a552 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -1238,7 +1238,7 @@ static PyMethodDef magick_Image_methods[] = { }, {"quantize", (PyCFunction)magick_Image_quantize, METH_VARARGS, - "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n nalyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." + "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n analyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." }, {NULL} /* Sentinel */ From e6ff759719124c6f2ea25d936b7794ddff197a03 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 31 Aug 2012 22:56:43 +0530 Subject: [PATCH 09/39] Make create_upload_path a utility function --- src/calibre/devices/__init__.py | 87 ++++++++++++++++++++++++++++- src/calibre/devices/usbms/device.py | 82 ++++----------------------- 2 files changed, 96 insertions(+), 73 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index ab772c6905..abe7d98b9f 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' Device drivers. ''' -import sys, time, pprint, operator, re +import sys, time, pprint, operator, re, os from functools import partial from StringIO import StringIO @@ -50,6 +50,91 @@ def build_template_regexp(template): template = u'{title} - {authors}' return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') +def create_upload_path(mdata, fname, template, sanitize, + prefix_path='', + path_type=os.path, + maxlen=250, + use_subdirs=True, + news_in_folder=True, + filename_callback=lambda x, y:x, + sanitize_path_components=lambda x: x + ): + from calibre.library.save_to_disk import get_components, config + from calibre.utils.filenames import shorten_components_to + + special_tag = None + if mdata.tags: + for t in mdata.tags: + if t.startswith(_('News')) or t.startswith('/'): + special_tag = t + break + + if mdata.tags and _('News') in mdata.tags: + try: + p = mdata.pubdate + date = (p.year, p.month, p.day) + except: + today = time.localtime() + date = (today[0], today[1], today[2]) + template = u"{title}_%d-%d-%d" % date + + fname = sanitize(fname) + ext = path_type.splitext(fname)[1] + + opts = config().parse() + if not isinstance(template, unicode): + template = template.decode('utf-8') + app_id = str(getattr(mdata, 'application_id', '')) + id_ = mdata.get('id', fname) + extra_components = get_components(template, mdata, id_, + timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) + if not extra_components: + extra_components.append(sanitize(filename_callback(fname, + mdata))) + else: + extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata)) + + if extra_components[-1] and extra_components[-1][0] in ('.', '_'): + extra_components[-1] = 'x' + extra_components[-1][1:] + + if special_tag is not None: + name = extra_components[-1] + extra_components = [] + tag = special_tag + if tag.startswith(_('News')): + if news_in_folder: + extra_components.append('News') + else: + for c in tag.split('/'): + c = sanitize(c) + if not c: continue + extra_components.append(c) + extra_components.append(name) + + if not use_subdirs: + extra_components = extra_components[-1:] + + def remove_trailing_periods(x): + ans = x + while ans.endswith('.'): + ans = ans[:-1].strip() + if not ans: + ans = 'x' + return ans + + extra_components = list(map(remove_trailing_periods, extra_components)) + if prefix_path: + prefix_path = path_type.abspath(prefix_path) + components = shorten_components_to(maxlen - len(prefix_path), extra_components) + components = sanitize_path_components(components) + if prefix_path: + filepath = path_type.join(prefix_path, *components) + else: + filepath = path_type.join(*components) + + return filepath + + def get_connected_device(): from calibre.customize.ui import device_plugins from calibre.devices.scanner import DeviceScanner diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 600916128d..4d4b198de0 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -19,7 +19,7 @@ from calibre.devices.errors import (DeviceError, FreeSpaceError, WrongDestinationError) from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins -from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to +from calibre.utils.filenames import ascii_filename as sanitize if isosx: usbobserver, usbobserver_err = plugins['usbobserver'] @@ -1052,78 +1052,16 @@ class Device(DeviceConfig, DevicePlugin): pass def create_upload_path(self, path, mdata, fname, create_dirs=True): - path = os.path.abspath(path) - maxlen = self.MAX_PATH_LEN - - special_tag = None - if mdata.tags: - for t in mdata.tags: - if t.startswith(_('News')) or t.startswith('/'): - special_tag = t - break - + from calibre.devices import create_upload_path settings = self.settings() - template = self.save_template() - if mdata.tags and _('News') in mdata.tags: - try: - p = mdata.pubdate - date = (p.year, p.month, p.day) - except: - today = time.localtime() - date = (today[0], today[1], today[2]) - template = "{title}_%d-%d-%d" % date - use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs - - fname = sanitize(fname) - ext = os.path.splitext(fname)[1] - - from calibre.library.save_to_disk import get_components - from calibre.library.save_to_disk import config - opts = config().parse() - if not isinstance(template, unicode): - template = template.decode('utf-8') - app_id = str(getattr(mdata, 'application_id', '')) - id_ = mdata.get('id', fname) - extra_components = get_components(template, mdata, id_, - timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1) - if not extra_components: - extra_components.append(sanitize(self.filename_callback(fname, - mdata))) - else: - extra_components[-1] = sanitize(self.filename_callback(extra_components[-1]+ext, mdata)) - - if extra_components[-1] and extra_components[-1][0] in ('.', '_'): - extra_components[-1] = 'x' + extra_components[-1][1:] - - if special_tag is not None: - name = extra_components[-1] - extra_components = [] - tag = special_tag - if tag.startswith(_('News')): - if self.NEWS_IN_FOLDER: - extra_components.append('News') - else: - for c in tag.split('/'): - c = sanitize(c) - if not c: continue - extra_components.append(c) - extra_components.append(name) - - if not use_subdirs: - extra_components = extra_components[-1:] - - def remove_trailing_periods(x): - ans = x - while ans.endswith('.'): - ans = ans[:-1].strip() - if not ans: - ans = 'x' - return ans - - extra_components = list(map(remove_trailing_periods, extra_components)) - components = shorten_components_to(maxlen - len(path), extra_components) - components = self.sanitize_path_components(components) - filepath = os.path.join(path, *components) + filepath = create_upload_path(mdata, fname, self.save_template(), sanitize, + prefix_path=os.path.abspath(path), + maxlen=self.MAX_PATH_LEN, + use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs, + news_in_folder = self.NEWS_IN_FOLDER, + filename_callback=self.filename_callback, + sanitize_path_components=self.sanitize_path_components + ) filedir = os.path.dirname(filepath) if create_dirs and not os.path.exists(filedir): From 30b3be2f0b0fe1384a7a84a5e28c64c4841f59a0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 00:37:47 +0530 Subject: [PATCH 10/39] ImageMagick: get and set image color depth --- src/calibre/utils/magick/magick.c | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index f01af3a552..94ac4dae83 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -1104,6 +1104,40 @@ magick_Image_type_setter(magick_Image *self, PyObject *val, void *closure) { // }}} +// Image.depth {{{ +static PyObject * +magick_Image_depth_getter(magick_Image *self, void *closure) { + NULL_CHECK(NULL) + + return Py_BuildValue("n", MagickGetImageDepth(self->wand)); +} + +static int +magick_Image_depth_setter(magick_Image *self, PyObject *val, void *closure) { + size_t depth; + + NULL_CHECK(-1) + + if (val == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete image depth"); + return -1; + } + + if (!PyInt_Check(val)) { + PyErr_SetString(PyExc_TypeError, "Depth must be an integer"); + return -1; + } + + depth = (size_t)PyInt_AsSsize_t(val); + if (!MagickSetImageDepth(self->wand, depth)) { + PyErr_Format(PyExc_ValueError, "Could not set image depth to %lu", depth); + return -1; + } + + return 0; +} + +// }}} // Image.destroy {{{ static PyObject * @@ -1260,6 +1294,12 @@ static PyGetSetDef magick_Image_getsetters[] = { (char *)"the image type: UndefinedType, BilevelType, GrayscaleType, GrayscaleMatteType, PaletteType, PaletteMatteType, TrueColorType, TrueColorMatteType, ColorSeparationType, ColorSeparationMatteType, or OptimizeType.", NULL}, + {(char *)"depth", + (getter)magick_Image_depth_getter, (setter)magick_Image_depth_setter, + (char *)"the image depth.", + NULL}, + + {NULL} /* Sentinel */ }; From b562f0324ce7a8d3e1d2a95b06b30b1cfbf92240 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 00:39:10 +0530 Subject: [PATCH 11/39] ... --- src/calibre/utils/magick/magick.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/utils/magick/magick.c b/src/calibre/utils/magick/magick.c index 94ac4dae83..6fbee2c77d 100644 --- a/src/calibre/utils/magick/magick.c +++ b/src/calibre/utils/magick/magick.c @@ -1138,6 +1138,7 @@ magick_Image_depth_setter(magick_Image *self, PyObject *val, void *closure) { } // }}} + // Image.destroy {{{ static PyObject * From f4ade0dc3bb8a4517b3a18f387251d7cdd3a7888 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 09:31:07 +0530 Subject: [PATCH 12/39] ... --- src/calibre/devices/__init__.py | 2 -- src/calibre/devices/mtp/base.py | 8 ++++++-- src/calibre/devices/mtp/driver.py | 16 +++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index abe7d98b9f..e67c17c063 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -123,8 +123,6 @@ def create_upload_path(mdata, fname, template, sanitize, return ans extra_components = list(map(remove_trailing_periods, extra_components)) - if prefix_path: - prefix_path = path_type.abspath(prefix_path) components = shorten_components_to(maxlen - len(prefix_path), extra_components) components = sanitize_path_components(components) if prefix_path: diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 346a44e6e8..865eeefb37 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -61,11 +61,15 @@ class MTPDeviceBase(DevicePlugin): def build_template_regexp(self): from calibre.devices import build_template_regexp - # TODO: Use the device specific template here - return build_template_regexp(self.default_save_template) + return build_template_regexp(self.save_template) @property def default_save_template(cls): from calibre.library.save_to_disk import config return config().parse().send_template + @property + def save_template(self): + # TODO: Use the device specific template here + return self.default_save_template + diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 7ba02e2aaa..2ec9e4e586 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, pprint, traceback +import json, pprint, traceback, posixpath from io import BytesIO from calibre import prints @@ -29,6 +29,8 @@ class MTP_DEVICE(BASE): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' CAN_SET_METADATA = [] + NEWS_IN_FOLDER = True + MAX_PATH_LEN = 230 def open(self, devices, library_uuid): self.current_library_uuid = library_uuid @@ -181,6 +183,18 @@ class MTP_DEVICE(BASE): self.put_file(storage, self.METADATA_CACHE, stream, size) # }}} + def create_upload_path(self, path, mdata, fname): + from calibre.devices import create_upload_path + from calibre.utils.filenames import ascii_filename as sanitize + filepath = create_upload_path(mdata, fname, self.save_template, sanitize, + prefix_path=path, + path_type=posixpath, + maxlen=self.MAX_PATH_LEN, + use_subdirs = True, + news_in_folder = self.NEWS_IN_FOLDER, + ) + return tuple(x.lower() for x in filepath.split('/')) + if __name__ == '__main__': dev = MTP_DEVICE(None) dev.startup() From c8b8825b1819b7f0ec16cd2e8d67b8f8cabdf562 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 09:48:20 +0530 Subject: [PATCH 13/39] Add an option under Preferences->Look & Feel->Book Details to hide the cover in the book details panel --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/book_details.py | 6 ++- src/calibre/gui2/preferences/look_feel.py | 1 + src/calibre/gui2/preferences/look_feel.ui | 55 +++++++++++++---------- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 00f5bef03d..8f275ec065 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -101,6 +101,7 @@ gprefs.defaults['auto_add_auto_convert'] = True gprefs.defaults['ui_style'] = 'calibre' if iswindows or isosx else 'system' gprefs.defaults['tag_browser_old_look'] = False gprefs.defaults['book_list_tooltips'] = True +gprefs.defaults['bd_show_cover'] = True # }}} NONE = QVariant() #: Null value to return from the data function of item models diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 3e20f8c67c..bf5fbe77bd 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -297,7 +297,8 @@ class CoverView(QWidget): # {{{ self.pixmap = self.default_pixmap self.do_layout() self.update() - if not same_item and not config['disable_animations']: + if (not same_item and not config['disable_animations'] and + self.isVisible()): self.animation.start() def paintEvent(self, event): @@ -512,6 +513,7 @@ class DetailsLayout(QLayout): # {{{ self.do_layout(r) def cover_height(self, r): + if not self._children[0].widget().isVisible(): return 0 mh = min(int(r.height()/2.), int(4/3. * r.width())+1) try: ph = self._children[0].widget().pixmap.height() @@ -522,6 +524,7 @@ class DetailsLayout(QLayout): # {{{ return mh def cover_width(self, r): + if not self._children[0].widget().isVisible(): return 0 mw = 1 + int(3/4. * r.height()) try: pw = self._children[0].widget().pixmap.width() @@ -660,6 +663,7 @@ class BookDetails(QWidget): # {{{ self.update_layout() def update_layout(self): + self.cover_view.setVisible(gprefs['bd_show_cover']) self._layout.do_layout(self.rect()) self.cover_view.update_tooltip(self.current_path) diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index 8ca6b96379..65e6ab6d9f 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -106,6 +106,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): 'calibre')]) r('book_list_tooltips', gprefs) r('tag_browser_old_look', gprefs, restart_required=True) + r('bd_show_cover', gprefs) r('cover_flow_queue_length', config, restart_required=True) diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index bb60f3db2a..2a4397b22d 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -212,19 +212,32 @@ Book Details - + + + + Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. + + + true + + + + + + + Use &Roman numerals for series + + + true + + + + Select displayed metadata - - - - true - - - @@ -247,6 +260,13 @@ + + + + true + + + @@ -288,23 +308,10 @@ Manage Authors. You can use the values {author} and - - + + - Use &Roman numerals for series - - - true - - - - - - - Note that <b>comments</b> will always be displayed at the end, regardless of the position you assign here. - - - true + Show &cover in the book details panel From bc959778e4bd8c06ae27e2f6cc9b36f22a6e53da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 11:48:28 +0530 Subject: [PATCH 14/39] ... --- src/calibre/devices/mtp/base.py | 5 ----- src/calibre/devices/mtp/driver.py | 16 +++++++++------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 865eeefb37..516b68ae1e 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -32,11 +32,6 @@ class MTPDeviceBase(DevicePlugin): author = 'Kovid Goyal' version = (1, 0, 0) - THUMBNAIL_HEIGHT = 128 - CAN_SET_METADATA = [] - - BACKLOADING_ERROR_MESSAGE = None - def __init__(self, *args, **kwargs): DevicePlugin.__init__(self, *args, **kwargs) self.progress_reporter = None diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 2ec9e4e586..72e8df9df8 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, pprint, traceback, posixpath +import json, traceback, posixpath, importlib from io import BytesIO from calibre import prints @@ -17,12 +17,8 @@ from calibre.ptempfile import SpooledTemporaryFile from calibre.utils.config import from_json, to_json from calibre.utils.date import now, isoformat -if iswindows: - from calibre.devices.mtp.windows.driver import MTP_DEVICE as BASE - BASE -else: - from calibre.devices.mtp.unix.driver import MTP_DEVICE as BASE -pprint +BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%( + 'windows' if iswindows else 'unix')).MTP_DEVICE class MTP_DEVICE(BASE): @@ -31,6 +27,11 @@ class MTP_DEVICE(BASE): CAN_SET_METADATA = [] NEWS_IN_FOLDER = True MAX_PATH_LEN = 230 + THUMBNAIL_HEIGHT = 160 + THUMBNAIL_WIDTH = 120 + CAN_SET_METADATA = [] + BACKLOADING_ERROR_MESSAGE = None + MANAGES_DEVICE_PRESENCE = True def open(self, devices, library_uuid): self.current_library_uuid = library_uuid @@ -157,6 +158,7 @@ class MTP_DEVICE(BASE): self.report_progress(count/steps, _('Updating metadata cache on device')) self.write_metadata_cache(storage, bl) self.report_progress(1, _('Finished reading metadata from device')) + return bl def read_file_metadata(self, mtp_file): from calibre.ebooks.metadata.meta import get_metadata From 786729fa6f3dc35b01c792ab271fe3a925bed9fd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 12:02:19 +0530 Subject: [PATCH 15/39] ... --- src/calibre/devices/mtp/books.py | 4 +++- src/calibre/devices/mtp/driver.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py index c02923702e..2179c49a8a 100644 --- a/src/calibre/devices/mtp/books.py +++ b/src/calibre/devices/mtp/books.py @@ -12,6 +12,7 @@ import os from calibre.devices.interface import BookList as BL from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.book.json_codec import JsonCodec +from calibre.utils.date import utcnow class BookList(BL): @@ -26,8 +27,9 @@ class Book(Metadata): def __init__(self, storage_id, lpath, other=None): Metadata.__init__(self, _('Unknown'), other=other) self.storage_id, self.lpath = storage_id, lpath - self.lpath = self.lpath.replace(os.sep, '/') + self.lpath = self.path = self.lpath.replace(os.sep, '/') self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')]) + self.datetime = utcnow().timetuple() def matches_file(self, mtp_file): return (self.storage_id == mtp_file.storage_id and diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 72e8df9df8..385914a9c9 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -183,6 +183,16 @@ class MTP_DEVICE(BASE): size = stream.tell() stream.seek(0) self.put_file(storage, self.METADATA_CACHE, stream, size) + + def sync_booklists(self, booklists, end_session=True): + for bl in booklists: + if getattr(bl, 'storage_id', None) is None: + continue + storage = self.filesystem_cache.storage(bl.storage_id) + if storage is None: + continue + self.write_metadata_cache(storage, bl) + # }}} def create_upload_path(self, path, mdata, fname): From 5601852363ede69773d1ea3bbe39c11a9c5c5537 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 12:54:12 +0530 Subject: [PATCH 16/39] Enable detection of MTP devices in the GUI and with ebook-device, with a tweak. Note that MTP support is not yet completed. --- src/calibre/customize/builtins.py | 7 +++- src/calibre/devices/cli.py | 20 +++++----- src/calibre/devices/interface.py | 41 +++++++++++++++++++ src/calibre/gui2/device.py | 65 +++++++++++++++++++++++-------- 4 files changed, 104 insertions(+), 29 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 469195627c..c7dc6a5b95 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -675,7 +675,6 @@ from calibre.devices.bambook.driver import BAMBOOK from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX from calibre.devices.smart_device_app.driver import SMART_DEVICE_APP - # Order here matters. The first matched device is the one used. plugins += [ HANLINV3, @@ -749,6 +748,12 @@ plugins += [ SMART_DEVICE_APP, USER_DEFINED, ] + +from calibre.utils.config_base import tweaks +if tweaks.get('test_mtp_driver', False): + from calibre.devices.mtp.driver import MTP_DEVICE + plugins.append(MTP_DEVICE) + # }}} # New metadata download plugins {{{ diff --git a/src/calibre/devices/cli.py b/src/calibre/devices/cli.py index 95181bf639..c7b105998d 100755 --- a/src/calibre/devices/cli.py +++ b/src/calibre/devices/cli.py @@ -9,7 +9,7 @@ For usage information run the script. import StringIO, sys, time, os from optparse import OptionParser -from calibre import __version__, __appname__ +from calibre import __version__, __appname__, human_readable from calibre.devices.errors import PathError from calibre.utils.terminfo import TerminalController from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked @@ -18,16 +18,6 @@ from calibre.devices.scanner import DeviceScanner MINIMUM_COL_WIDTH = 12 #: Minimum width of columns in ls output -def human_readable(size): - """ Convert a size in bytes into a human readle form """ - if size < 1024: divisor, suffix = 1, "" - elif size < 1024*1024: divisor, suffix = 1024., "K" - elif size < 1024*1024*1024: divisor, suffix = 1024*1024, "M" - elif size < 1024*1024*1024*1024: divisor, suffix = 1024*1024, "G" - size = str(size/divisor) - if size.find(".") > -1: size = size[:size.find(".")+2] - return size + suffix - class FileFormatter(object): def __init__(self, file, term): self.term = term @@ -207,11 +197,19 @@ def main(): scanner = DeviceScanner() scanner.scan() connected_devices = [] + for d in device_plugins(): try: d.startup() except: print ('Startup failed for device plugin: %s'%d) + if d.MANAGES_DEVICE_PRESENCE: + cd = d.detect_managed_devices(scanner.devices) + if cd is not None: + connected_devices.append((cd, d)) + dev = d + break + continue ok, det = scanner.is_device_connected(d) if ok: dev = d diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 2c5f60ecfe..6d859c8f89 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -81,6 +81,19 @@ class DevicePlugin(Plugin): #: by. NUKE_COMMENTS = None + #: If True indicates that this driver completely manages device detection, + #: ejecting and so forth. If you set this to True, you *must* implement the + #: detect_managed_devices and debug_managed_device_detection methods. + #: A driver with this set to true is responsible for detection of devices, + #: managing a blacklist of devices, a list of ejected devices and so forth. + #: calibre will periodically call the detect_managed_devices() method and + #: is it returns a detected device, calibre will call open(). open() will + #: be called every time a device is returned even is previous calls to open() + #: failed, therefore the driver must maintain its own blacklist of failed + #: devices. Similarly, when ejecting, calibre will call eject() and then + #: assuming the next call to detect_managed_devices() returns None, it will + #: call post_yank_cleanup(). + MANAGES_DEVICE_PRESENCE = False @classmethod def get_gui_name(cls): @@ -196,6 +209,34 @@ class DevicePlugin(Plugin): return True, dev return False, None + def detect_managed_devices(self, devices_on_system, force_refresh=False): + ''' + Called only if MANAGES_DEVICE_PRESENCE is True. + + Scan for devices that this driver can handle. Should return a device + object if a device is found. This object will be passed to the open() + method as the connected_device. If no device is found, return None. + + This method is called periodically by the GUI, so make sure it is not + too resource intensive. Use a cache to avoid repeatedly scanning the + system. + + :param devices_on_system: Set of USB devices found on the system. + + :param force_refresh: If True and the driver uses a cache to prevent + repeated scanning, the cache must be flushed. + ''' + raise NotImplementedError() + + def debug_managed_device_detection(self, devices_on_system, output): + ''' + Called only if MANAGES_DEVICE_PRESENCE is True. + + Should write information about the devices detected on the system to + output, which is a file like object. + ''' + raise NotImplementedError() + # }}} def reset(self, key='-1', log_packets=False, report_progress=None, diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 03e1932035..d5879042b4 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -128,6 +128,10 @@ class DeviceManager(Thread): # {{{ self.setDaemon(True) # [Device driver, Showing in GUI, Ejected] self.devices = list(device_plugins()) + self.managed_devices = [x for x in self.devices if + not x.MANAGES_DEVICE_PRESENCE] + self.unmanaged_devices = [x for x in self.devices if + x.MANAGES_DEVICE_PRESENCE] self.sleep_time = sleep_time self.connected_slot = connected_slot self.jobs = Queue.Queue(0) @@ -182,12 +186,15 @@ class DeviceManager(Thread): # {{{ prints('Unable to open device', str(dev)) prints(tb) continue - self.connected_device = dev - self.connected_device_kind = device_kind - self.connected_slot(True, device_kind) + self.after_device_connect(dev, device_kind) return True return False + def after_device_connect(self, dev, device_kind): + self.connected_device = dev + self.connected_device_kind = device_kind + self.connected_slot(True, device_kind) + def connected_device_removed(self): while True: try: @@ -215,22 +222,45 @@ class DeviceManager(Thread): # {{{ def detect_device(self): self.scanner.scan() + if self.is_device_connected: - connected, detected_device = \ - self.scanner.is_device_connected(self.connected_device, - only_presence=True) - if not connected: - if DEBUG: - # Allow the device subsystem to output debugging info about - # why it thinks the device is not connected. Used, for e.g. - # in the can_handle() method of the T1 driver + if self.connected_device.MANAGES_DEVICE_PRESENCE: + cd = self.connected_device.detect_managed_devices(self.scanner.devices) + if cd is None: + self.connected_device_removed() + else: + connected, detected_device = \ self.scanner.is_device_connected(self.connected_device, - only_presence=True, debug=True) - self.connected_device_removed() + only_presence=True) + if not connected: + if DEBUG: + # Allow the device subsystem to output debugging info about + # why it thinks the device is not connected. Used, for e.g. + # in the can_handle() method of the T1 driver + self.scanner.is_device_connected(self.connected_device, + only_presence=True, debug=True) + self.connected_device_removed() else: + for dev in self.unmanaged_devices: + try: + cd = dev.detect_managed_devices(self.scanner.devices) + except: + prints('Error during device detection for %s:'%dev) + traceback.print_exc() + else: + if cd is not None: + try: + dev.open(cd, self.current_library_uuid) + except: + prints('Error while trying to open %s (Driver: %s)'% + (cd, dev)) + traceback.print_exc() + else: + self.after_device_connect(dev, 'unmanaged-device') + return try: possibly_connected_devices = [] - for device in self.devices: + for device in self.managed_devices: if device in self.ejected_devices: continue try: @@ -248,7 +278,7 @@ class DeviceManager(Thread): # {{{ prints('Connect to device failed, retrying in 5 seconds...') time.sleep(5) if not self.do_connect(possibly_connected_devices, - device_kind='usb'): + device_kind='device'): if DEBUG: prints('Device connect failed again, giving up') except OpenFailed as e: @@ -264,9 +294,10 @@ class DeviceManager(Thread): # {{{ # disconnect a device def umount_device(self, *args): if self.is_device_connected and not self.job_manager.has_device_jobs(): - if self.connected_device_kind == 'device': + if self.connected_device_kind in {'unmanaged-device', 'device'}: self.connected_device.eject() - self.ejected_devices.add(self.connected_device) + if self.connected_device_kind != 'unmanaged-device': + self.ejected_devices.add(self.connected_device) self.connected_slot(False, self.connected_device_kind) elif hasattr(self.connected_device, 'unmount_device'): # As we are on the wrong thread, this call must *not* do From 63e6014edcdcc3509508466e4db801c99096e785 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 13:11:24 +0530 Subject: [PATCH 17/39] ... --- src/calibre/devices/interface.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 6d859c8f89..7512446905 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -234,6 +234,9 @@ class DevicePlugin(Plugin): Should write information about the devices detected on the system to output, which is a file like object. + + Should return True if a device was detected and successfully opened, + otherwise False. ''' raise NotImplementedError() From 023b94760868afa0023f072d299dd22a78ebebaf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 13:41:37 +0530 Subject: [PATCH 18/39] Debug device detection for MTP devices --- src/calibre/devices/__init__.py | 103 ++++++++++++---------- src/calibre/devices/mtp/unix/driver.py | 2 + src/calibre/devices/mtp/windows/driver.py | 52 ++++++++++- 3 files changed, 109 insertions(+), 48 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index e67c17c063..89d0e4e026 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -221,54 +221,65 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None): out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in devplugins]))) out(' ') - out('Looking for devices...') + found_dev = False for dev in devplugins: - connected, det = s.is_device_connected(dev, debug=True) - if connected: - out('\t\tDetected possible device', dev.__class__.__name__) - connected_devices.append((dev, det)) - - out(' ') - errors = {} - success = False - out('Devices possibly connected:', end=' ') - for dev, det in connected_devices: - out(dev.name, end=', ') - if not connected_devices: - out('None', end='') - out(' ') - for dev, det in connected_devices: - out('Trying to open', dev.name, '...', end=' ') - try: - dev.reset(detected_device=det) - dev.open(det, None) - out('OK') - except: - import traceback - errors[dev] = traceback.format_exc() - out('failed') - continue - success = True - if hasattr(dev, '_main_prefix'): - out('Main memory:', repr(dev._main_prefix)) - out('Total space:', dev.total_space()) - break - if not success and errors: - out('Opening of the following devices failed') - for dev,msg in errors.items(): - out(dev) - out(msg) - out(' ') - - if ioreg is not None: - ioreg = 'IOREG Output\n'+ioreg + if not dev.MANAGES_DEVICE_PRESENCE: continue + out('Looking for devices of type:', dev.__class__.__name__) + if dev.debug_managed_device_detection(s.devices, buf): + found_dev = True + break out(' ') - if ioreg_to_tmp: - open('/tmp/ioreg.txt', 'wb').write(ioreg) - out('Dont forget to send the contents of /tmp/ioreg.txt') - out('You can open it with the command: open /tmp/ioreg.txt') - else: - out(ioreg) + + if not found_dev: + out('Looking for devices...') + for dev in devplugins: + if dev.MANAGES_DEVICE_PRESENCE: continue + connected, det = s.is_device_connected(dev, debug=True) + if connected: + out('\t\tDetected possible device', dev.__class__.__name__) + connected_devices.append((dev, det)) + + out(' ') + errors = {} + success = False + out('Devices possibly connected:', end=' ') + for dev, det in connected_devices: + out(dev.name, end=', ') + if not connected_devices: + out('None', end='') + out(' ') + for dev, det in connected_devices: + out('Trying to open', dev.name, '...', end=' ') + try: + dev.reset(detected_device=det) + dev.open(det, None) + out('OK') + except: + import traceback + errors[dev] = traceback.format_exc() + out('failed') + continue + success = True + if hasattr(dev, '_main_prefix'): + out('Main memory:', repr(dev._main_prefix)) + out('Total space:', dev.total_space()) + break + if not success and errors: + out('Opening of the following devices failed') + for dev,msg in errors.items(): + out(dev) + out(msg) + out(' ') + + if ioreg is not None: + ioreg = 'IOREG Output\n'+ioreg + out(' ') + if ioreg_to_tmp: + open('/tmp/ioreg.txt', 'wb').write(ioreg) + out('Dont forget to send the contents of /tmp/ioreg.txt') + out('You can open it with the command: open /tmp/ioreg.txt') + else: + out(ioreg) if hasattr(buf, 'getvalue'): return buf.getvalue().decode('utf-8') diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 9244ac198c..b4e8b44407 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -82,6 +82,8 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def debug_managed_device_detection(self, devices_on_system, output): + if self.currently_connected_dev is not None: + return True p = partial(prints, file=output) if self.libmtp is None: err = plugins['libmtp'][1] diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 191d69560d..abe38e5f7c 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -7,8 +7,8 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import time, threading -from functools import wraps +import time, threading, traceback +from functools import wraps, partial from future_builtins import zip from itertools import chain @@ -123,6 +123,54 @@ class MTP_DEVICE(MTPDeviceBase): return None + @same_thread + def debug_managed_device_detection(self, devices_on_system, output): + import pprint + p = partial(prints, file=output) + if self.currently_connected_pnp_id is not None: + return True + if self.wpd_error: + p('Cannot detect MTP devices') + p(self.wpd_error) + return False + try: + pnp_ids = frozenset(self.wpd.enumerate_devices()) + except: + p("Failed to get list of PNP ids on system") + p(traceback.format_exc()) + return False + + for pnp_id in pnp_ids: + try: + data = self.wpd.device_info(pnp_id) + except: + p('Failed to get data for device:', pnp_id) + p(traceback.format_exc()) + continue + protocol = data.get('protocol', '').lower() + if not protocol.startswith('mtp:'): continue + p('MTP device:', pnp_id) + p(pprint.pformat(data)) + if not self.is_suitable_wpd_device(data): + p('Not a suitable MTP device, ignoring\n') + continue + p('\nTrying to open:', pnp_id) + try: + self.open(pnp_id, 'debug-detection') + except: + p('Open failed:') + p(traceback.format_exc()) + continue + break + if self.currently_connected_pnp_id: + p('Opened', self.current_friendly_name, 'successfully') + p('Device info:') + p(pprint.pformat(self.dev.data)) + self.eject() + return True + p('No suitable MTP devices found') + return False + def is_suitable_wpd_device(self, devdata): # Check that protocol is MTP protocol = devdata.get('protocol', '').lower() From 1c3e62f35bf9527df9f0a218bd85272644fa4fdb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 14:51:52 +0530 Subject: [PATCH 19/39] MTP: Try to get modified date from the device --- src/calibre/devices/mtp/filesystem_cache.py | 10 +++++++++- src/calibre/devices/mtp/unix/libmtp.c | 3 ++- .../devices/mtp/windows/content_enumeration.cpp | 13 +++++++++++++ src/calibre/devices/mtp/windows/driver.py | 4 +++- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index ba2206d191..e4ef8ae898 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -11,8 +11,10 @@ import weakref, sys from collections import deque from operator import attrgetter from future_builtins import map +from datetime import datetime from calibre import human_readable, prints, force_unicode +from calibre.utils.date import local_tz from calibre.utils.icu import sort_key, lower from calibre.ebooks import BOOK_EXTENSIONS @@ -21,6 +23,8 @@ bexts = frozenset(BOOK_EXTENSIONS) class FileOrFolder(object): def __init__(self, entry, fs_cache): + self.all_storage_ids = fs_cache.all_storage_ids + self.object_id = entry['id'] self.is_folder = entry['is_folder'] self.storage_id = entry['storage_id'] @@ -31,7 +35,11 @@ class FileOrFolder(object): self.name = force_unicode(n, 'utf-8') self.persistent_id = entry.get('persistent_id', self.object_id) self.size = entry.get('size', 0) - self.all_storage_ids = fs_cache.all_storage_ids + md = entry.get('modified', 0) + try: + self.last_modified = datetime.fromtimestamp(md, local_tz) + except: + self.last_modified = datetime.fromtimestamp(0, local_tz) if self.storage_id not in self.all_storage_ids: raise ValueError('Storage id %s not valid for %s, valid values: %s'%(self.storage_id, diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 86c9349d20..bf07c73a35 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -121,12 +121,13 @@ static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, uns static PyObject* build_file_metadata(LIBMTP_file_t *nf, uint32_t storage_id) { PyObject *ans = NULL; - ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:O}", + ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:L, s:O}", "name", (unsigned long)nf->filename, "id", (unsigned long)nf->item_id, "parent_id", (unsigned long)nf->parent_id, "storage_id", (unsigned long)storage_id, "size", nf->filesize, + "modified", (PY_LONG_LONG)nf->modificationdate, "is_folder", (nf->filetype == LIBMTP_FILETYPE_FOLDER) ? Py_True : Py_False ); diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index e1f439926c..7186bbdcdb 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -34,6 +34,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { ADDPROP(WPD_OBJECT_ISHIDDEN); ADDPROP(WPD_OBJECT_CAN_DELETE); ADDPROP(WPD_OBJECT_SIZE); + ADDPROP(WPD_OBJECT_DATE_MODIFIED); return properties; @@ -81,6 +82,16 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py } } +static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) { + FLOAT val = 0; + PyObject *t; + + if (SUCCEEDED(properties->GetFloatValue(key, &val))) { + t = Py_BuildValue("d", (double)val); + if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); } + } +} + static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) { GUID guid = GUID_NULL; BOOL is_folder = 0; @@ -103,6 +114,8 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) { set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values); set_size_property(obj, WPD_OBJECT_SIZE, "size", values); + set_date_property(obj, WPD_OBJECT_DATE_MODIFIED, "modified", values); + } // }}} diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index abe38e5f7c..dbcad94fe0 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -248,7 +248,9 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = devdata.get('friendly_name', None) + self.current_friendly_name = devdata.get('friendly_name', + _('Unknown MTP device')) + self.currently_connected_pnp_id = connected_device @same_thread def get_basic_device_information(self): From 49d1fbad1805ed4c07e7faba6a567b0b457dc93d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 14:55:03 +0530 Subject: [PATCH 20/39] ... --- src/calibre/devices/mtp/windows/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index dbcad94fe0..aaadffbc68 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -21,7 +21,7 @@ from calibre.devices.mtp.base import MTPDeviceBase class ThreadingViolation(Exception): def __init__(self): - Exception.__init__('You cannot use the MTP driver from a thread other than the ' + Exception.__init__(self, 'You cannot use the MTP driver from a thread other than the ' ' thread in which startup() was called') def same_thread(func): From 9a110fb0d50a614b23738cec14a53365c443f35f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 14:55:26 +0530 Subject: [PATCH 21/39] ... --- src/calibre/devices/mtp/windows/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index aaadffbc68..92e9790734 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -21,7 +21,8 @@ from calibre.devices.mtp.base import MTPDeviceBase class ThreadingViolation(Exception): def __init__(self): - Exception.__init__(self, 'You cannot use the MTP driver from a thread other than the ' + Exception.__init__(self, + 'You cannot use the MTP driver from a thread other than the ' ' thread in which startup() was called') def same_thread(func): From d278180bde633cc280bfe8f7805ad15b3d88a0dd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 15:06:53 +0530 Subject: [PATCH 22/39] Windows: workaround for eject() not being called on the device thread --- src/calibre/devices/interface.py | 3 +++ src/calibre/devices/mtp/windows/driver.py | 24 ++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 7512446905..4777cafbe9 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -314,6 +314,9 @@ class DevicePlugin(Plugin): ''' Un-mount / eject the device from the OS. This does not check if there are pending GUI jobs that need to communicate with the device. + + NOTE: That this method may not be called on the same thread as the rest + of the device methods. ''' raise NotImplementedError() diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 92e9790734..faa5296547 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -51,6 +51,7 @@ class MTP_DEVICE(MTPDeviceBase): self._main_id = self._carda_id = self._cardb_id = None self.start_thread = None self._filesystem_cache = None + self.eject_dev_on_next_scan = False def startup(self): self.start_thread = threading.current_thread() @@ -75,6 +76,10 @@ class MTP_DEVICE(MTPDeviceBase): @same_thread def detect_managed_devices(self, devices_on_system, force_refresh=False): if self.wpd is None: return None + if self.eject_dev_on_next_scan: + self.eject_dev_on_next_scan = False + if self.currently_connected_pnp_id is not None: + self.do_eject() devices_on_system = frozenset(devices_on_system) if (force_refresh or @@ -213,19 +218,24 @@ class MTP_DEVICE(MTPDeviceBase): return self._filesystem_cache @same_thread - def post_yank_cleanup(self): - self.currently_connected_pnp_id = self.current_friendly_name = None - self._main_id = self._carda_id = self._cardb_id = None - self.dev = self._filesystem_cache = None - - @same_thread - def eject(self): + def do_eject(self): if self.currently_connected_pnp_id is None: return self.ejected_devices.add(self.currently_connected_pnp_id) self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None self.dev = self._filesystem_cache = None + + @same_thread + def post_yank_cleanup(self): + self.currently_connected_pnp_id = self.current_friendly_name = None + self._main_id = self._carda_id = self._cardb_id = None + self.dev = self._filesystem_cache = None + + def eject(self): + if self.currently_connected_pnp_id is None: return + self.eject_dev_on_next_scan = True + @same_thread def open(self, connected_device, library_uuid): self.dev = self._filesystem_cache = None From 7b2a947a86b66612a9244d420d579dbfb76db5f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 15:22:49 +0530 Subject: [PATCH 23/39] ... --- src/calibre/devices/mtp/driver.py | 2 ++ src/calibre/devices/mtp/filesystem_cache.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 385914a9c9..4b26e23ef8 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -129,6 +129,7 @@ class MTP_DEVICE(BASE): cached_metadata = bl[idx] del relpath_cache[relpath] if cached_metadata.size == mtp_file.size: + cached_metadata.datetime = mtp_file.last_modified.timetuple() debug('Using cached metadata for', '/'.join(mtp_file.full_path)) continue # No need to update metadata @@ -148,6 +149,7 @@ class MTP_DEVICE(BASE): '/'.join(mtp_file.full_path)) traceback.print_exc() book.size = mtp_file.size + book.datetime = mtp_file.last_modified.timetuple() # Remove books in the cache that no longer exist for idx in sorted(relpath_cache.itervalues(), reverse=True): diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index e4ef8ae898..6aab711199 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -14,7 +14,7 @@ from future_builtins import map from datetime import datetime from calibre import human_readable, prints, force_unicode -from calibre.utils.date import local_tz +from calibre.utils.date import local_tz, as_utc from calibre.utils.icu import sort_key, lower from calibre.ebooks import BOOK_EXTENSIONS @@ -40,6 +40,7 @@ class FileOrFolder(object): self.last_modified = datetime.fromtimestamp(md, local_tz) except: self.last_modified = datetime.fromtimestamp(0, local_tz) + self.last_modified = as_utc(self.last_modified) if self.storage_id not in self.all_storage_ids: raise ValueError('Storage id %s not valid for %s, valid values: %s'%(self.storage_id, From 4785359e6ab1993fbc7e7f134e4882b04d82c7ab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 15:33:33 +0530 Subject: [PATCH 24/39] ... --- src/calibre/devices/mtp/books.py | 1 + src/calibre/devices/mtp/driver.py | 6 +++--- src/calibre/devices/mtp/test.py | 6 +++--- src/calibre/devices/mtp/unix/driver.py | 2 +- src/calibre/devices/mtp/windows/driver.py | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py index 2179c49a8a..73e483f19e 100644 --- a/src/calibre/devices/mtp/books.py +++ b/src/calibre/devices/mtp/books.py @@ -30,6 +30,7 @@ class Book(Metadata): self.lpath = self.path = self.lpath.replace(os.sep, '/') self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')]) self.datetime = utcnow().timetuple() + self.thumbail = None def matches_file(self, mtp_file): return (self.storage_id == mtp_file.storage_id and diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 4b26e23ef8..13e8394288 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -43,7 +43,7 @@ class MTP_DEVICE(BASE): f = storage.find_path((self.DRIVEINFO,)) dinfo = {} if f is not None: - stream = self.get_file(f) + stream = self.get_mtp_file(f) try: dinfo = json.load(stream, object_hook=from_json) except: @@ -114,7 +114,7 @@ class MTP_DEVICE(BASE): if cache is not None: json_codec = JSONCodec() try: - stream = self.get_file(cache) + stream = self.get_mtp_file(cache) json_codec.decode_from_file(stream, bl, Book, sid) except: need_sync = True @@ -166,7 +166,7 @@ class MTP_DEVICE(BASE): from calibre.ebooks.metadata.meta import get_metadata from calibre.customize.ui import quick_metadata ext = mtp_file.name.rpartition('.')[-1].lower() - stream = self.get_file(mtp_file) + stream = self.get_mtp_file(mtp_file) with quick_metadata: return get_metadata(stream, stream_type=ext, force_read_metadata=True, diff --git a/src/calibre/devices/mtp/test.py b/src/calibre/devices/mtp/test.py index 0563708ea4..c273bac5e0 100644 --- a/src/calibre/devices/mtp/test.py +++ b/src/calibre/devices/mtp/test.py @@ -128,7 +128,7 @@ class TestDeviceInteraction(unittest.TestCase): raw2 = io.BytesIO() pc = ProgressCallback() - self.dev.get_file(f, raw2, callback=pc) + self.dev.get_mtp_file(f, raw2, callback=pc) self.assertEqual(raw.getvalue(), raw2.getvalue()) self.assertTrue(pc.end_called, msg='Progress callback not called with equal values (get_file)') @@ -162,7 +162,7 @@ class TestDeviceInteraction(unittest.TestCase): self.assertEqual(f.storage_id, self.storage.storage_id) raw2 = io.BytesIO() - self.dev.get_file(f, raw2) + self.dev.get_mtp_file(f, raw2) self.assertEqual(raw.getvalue(), raw2.getvalue()) def measure_memory_usage(self, repetitions, func, *args, **kwargs): @@ -226,7 +226,7 @@ class TestDeviceInteraction(unittest.TestCase): def get_file(f): raw = io.BytesIO() pc = ProgressCallback() - self.dev.get_file(f, raw, callback=pc) + self.dev.get_mtp_file(f, raw, callback=pc) raw.truncate(0) del raw del pc diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index b4e8b44407..338913114f 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -273,7 +273,7 @@ class MTP_DEVICE(MTPDeviceBase): return parent.add_child(ans) @synchronous - def get_file(self, f, stream=None, callback=None): + def get_mtp_file(self, f, stream=None, callback=None): if f.is_folder: raise ValueError('%s if a folder'%(f.full_path,)) if stream is None: diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index faa5296547..7c15797ef6 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -293,7 +293,7 @@ class MTP_DEVICE(MTPDeviceBase): return tuple(ans) @same_thread - def get_file(self, f, stream=None, callback=None): + def get_mtp_file(self, f, stream=None, callback=None): if f.is_folder: raise ValueError('%s if a folder'%(f.full_path,)) if stream is None: From b9737b96593e8bef40660cf33889825f24cb63fd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 21:57:09 +0530 Subject: [PATCH 25/39] MTP: Implement get_file() --- src/calibre/devices/mtp/driver.py | 6 ++++++ src/calibre/devices/mtp/filesystem_cache.py | 21 ++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 13e8394288..9ac0c3d31a 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -130,6 +130,7 @@ class MTP_DEVICE(BASE): del relpath_cache[relpath] if cached_metadata.size == mtp_file.size: cached_metadata.datetime = mtp_file.last_modified.timetuple() + cached_metadata.path = mtp_file.mtp_id_path debug('Using cached metadata for', '/'.join(mtp_file.full_path)) continue # No need to update metadata @@ -150,6 +151,7 @@ class MTP_DEVICE(BASE): traceback.print_exc() book.size = mtp_file.size book.datetime = mtp_file.last_modified.timetuple() + book.path = mtp_file.mtp_id_path # Remove books in the cache that no longer exist for idx in sorted(relpath_cache.itervalues(), reverse=True): @@ -197,6 +199,10 @@ class MTP_DEVICE(BASE): # }}} + def get_file(self, path, outfile, end_session=True): + f = self.filesystem_cache.resolve_mtp_id_path(path) + self.get_mtp_file(f, outfile) + def create_upload_path(self, path, mdata, fname): from calibre.devices import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index 6aab711199..216e06031f 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import weakref, sys +import weakref, sys, json from collections import deque from operator import attrgetter from future_builtins import map @@ -166,6 +166,10 @@ class FileOrFolder(object): def mtp_relpath(self): return tuple(x.lower() for x in self.full_path[1:]) + @property + def mtp_id_path(self): + return 'mtp:::' + json.dumps(self.object_id) + ':::' + '/'.join(self.full_path) + class FilesystemCache(object): def __init__(self, all_storage, entries): @@ -215,4 +219,19 @@ class FilesystemCache(object): if x.storage_id == storage_id and x.is_ebook: yield x + def resolve_mtp_id_path(self, path): + if not path.startswith('mtp:::'): + raise ValueError('%s is not a valid MTP path'%path) + parts = path.split(':::') + if len(parts) < 3: + raise ValueError('%s is not a valid MTP path'%path) + try: + object_id = json.loads(parts[1]) + except: + raise ValueError('%s is not a valid MTP path'%path) + try: + return self.id_map[object_id] + except KeyError: + raise ValueError('No object found with MTP path: %s'%path) + From 2e66bd1aa58f141e973b7cc350db587ee394d4d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 23:00:40 +0530 Subject: [PATCH 26/39] MTP: Implement prepare_addable_books and refactor the GUI to run prepare_addable_books in the device thread --- src/calibre/devices/mtp/driver.py | 26 +++++++++++++-- src/calibre/gui2/actions/add.py | 53 +++++++++++++++++++++++++------ src/calibre/gui2/device.py | 8 +++++ 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index 9ac0c3d31a..8f8f4d119b 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,13 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, traceback, posixpath, importlib +import json, traceback, posixpath, importlib, os from io import BytesIO from calibre import prints from calibre.constants import iswindows, numeric_version from calibre.devices.mtp.base import debug -from calibre.ptempfile import SpooledTemporaryFile +from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory from calibre.utils.config import from_json, to_json from calibre.utils.date import now, isoformat @@ -199,10 +199,32 @@ class MTP_DEVICE(BASE): # }}} + # Get files from the device {{{ def get_file(self, path, outfile, end_session=True): f = self.filesystem_cache.resolve_mtp_id_path(path) self.get_mtp_file(f, outfile) + def prepare_addable_books(self, paths): + tdir = PersistentTemporaryDirectory('_prepare_mtp') + ans = [] + for path in paths: + try: + f = self.filesystem_cache.resolve_mtp_id_path(path) + except Exception as e: + ans.append((path, e, traceback.format_exc())) + continue + base = os.path.join(tdir, '%s'%f.object_id) + os.mkdir(base) + with open(os.path.join(base, f.name), 'wb') as out: + try: + self.get_mtp_file(f, out) + except Exception as e: + ans.append((path, e, traceback.format_exc())) + else: + ans.append(out.name) + return ans + # }}} + def create_upload_path(self, path, mdata, fname): from calibre.devices import create_upload_path from calibre.utils.filenames import ascii_filename as sanitize diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 9d15fa4ac8..ef7ed7a594 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -10,9 +10,9 @@ from functools import partial from PyQt4.Qt import QPixmap, QTimer - -from calibre.gui2 import error_dialog, choose_files, \ - choose_dir, warning_dialog, info_dialog +from calibre import as_unicode +from calibre.gui2 import (error_dialog, choose_files, choose_dir, + warning_dialog, info_dialog) from calibre.gui2.dialogs.add_empty_book import AddEmptyBookDialog from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2.widgets import IMAGE_EXTENSIONS @@ -400,12 +400,45 @@ class AddAction(InterfaceAction): d = error_dialog(self.gui, _('Add to library'), _('No book files found')) d.exec_() return - paths = self.gui.device_manager.device.prepare_addable_books(paths) - from calibre.gui2.add import Adder - self.__adder_func = partial(self._add_from_device_adder, on_card=None, - model=view.model()) - self._adder = Adder(self.gui, self.gui.library_view.model().db, - self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) - self._adder.add(paths) + + self.gui.device_manager.prepare_addable_books(self.Dispatcher(partial( + self.books_prepared, view)), paths) + self.bpd = ProgressDialog(_('Downloading books'), + msg=_('Downloading books from device'), parent=self.gui, + cancelable=False) + QTimer.singleShot(1000, self.show_bpd) + + def show_bpd(self): + if self.bpd is not None: + self.bpd.show() + + def books_prepared(self, view, job): + self.bpd.hide() + self.bpd = None + if job.exception is not None: + self.gui.device_job_exception(job) + return + paths = job.result + ok_paths = [x for x in paths if isinstance(x, basestring)] + failed_paths = [x for x in paths if isinstance(x, tuple)] + if failed_paths: + if not ok_paths: + msg = _('Could not download files from the device') + typ = error_dialog + else: + msg = _('Could not download some files from the device') + typ = warning_dialog + det_msg = [x[0]+ '\n ' + as_unicode(x[1]) for x in failed_paths] + det_msg = '\n\n'.join(det_msg) + typ(self.gui, _('Could not download files'), msg, det_msg=det_msg, + show=True) + + if ok_paths: + from calibre.gui2.add import Adder + self.__adder_func = partial(self._add_from_device_adder, on_card=None, + model=view.model()) + self._adder = Adder(self.gui, self.gui.library_view.model().db, + self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server) + self._adder.add(ok_paths) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index d5879042b4..5f9cb3e75c 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -443,6 +443,14 @@ class DeviceManager(Thread): # {{{ return self.create_job_step(self._books, done, description=_('Get list of books on device'), to_job=add_as_step_to_job) + def _prepare_addable_books(self, paths): + return self.device.prepare_addable_books(paths) + + def prepare_addable_books(self, done, paths, add_as_step_to_job=None): + return self.create_job_step(self._prepare_addable_books, done, args=[paths], + description=_('Prepare files for transfer from device'), + to_job=add_as_step_to_job) + def _annotations(self, path_map): return self.device.get_annotations(path_map) From 468a47eb5d8f573935ccd4ac7ec6cf256515fbaf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 23:07:10 +0530 Subject: [PATCH 27/39] ... --- src/calibre/gui2/device.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 5f9cb3e75c..98e42f4178 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -564,9 +564,8 @@ class DeviceManager(Thread): # {{{ to_job=add_as_step_to_job) def _view_book(self, path, target): - f = open(target, 'wb') - self.device.get_file(path, f) - f.close() + with open(target, 'wb') as f: + self.device.get_file(path, f) return target def view_book(self, done, path, target, add_as_step_to_job=None): From e61dbefc7f250edc955f3b15813e548ebf7a4d72 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 1 Sep 2012 23:12:18 +0530 Subject: [PATCH 28/39] ... --- src/calibre/devices/interface.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 4777cafbe9..d0b2611ead 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -543,6 +543,10 @@ class DevicePlugin(Plugin): ''' Given a list of paths, returns another list of paths. These paths point to addable versions of the books. + + If there is an error preparing a book, then instead of a path, the + position in the returned list for that book should be a three tuple: + (original_path, the exception instance, traceback) ''' return paths From 4bc92ec184f0c4430ff73242a17052097228b28d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 00:06:02 +0530 Subject: [PATCH 29/39] PDF Output: Fix page numbers in outline not always correct --- src/calibre/ebooks/pdf/outline_writer.py | 6 +++--- src/calibre/ebooks/pdf/writer.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/pdf/outline_writer.py b/src/calibre/ebooks/pdf/outline_writer.py index 4b2db84f9e..c89f2d9f41 100644 --- a/src/calibre/ebooks/pdf/outline_writer.py +++ b/src/calibre/ebooks/pdf/outline_writer.py @@ -35,12 +35,12 @@ class Outline(object): page, ypos = 0, 0 item = getattr(toc, 'outline_item_', None) if item is not None: + # First use the item URL without fragment + page, ypos = self.pos_map.get(item, {}).get(None, (0, 0)) if toc.fragment: amap = self.pos_map.get(item, None) if amap is not None: - page, ypos = amap.get(toc.fragment, (0, 0)) - else: - page, ypos = self.pos_map.get(item, {}).get(None, (0, 0)) + page, ypos = amap.get(toc.fragment, (page, ypos)) return page, ypos def add_children(self, toc, parent): diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index d37db69a81..92054e2f76 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -196,6 +196,7 @@ class PDFWriter(QObject): # {{{ self.insert_cover() self.render_succeeded = False + self.current_page_num = self.doc.page_count() self.combine_queue.append(os.path.join(self.tmp_path, 'qprinter_out.pdf')) self.first_page = True @@ -283,9 +284,13 @@ class PDFWriter(QObject): # {{{ paged_display.fit_images(); ''') mf = self.view.page().mainFrame() + start_page = self.current_page_num + if not self.first_page: + start_page += 1 while True: if not self.first_page: - self.printer.newPage() + if self.printer.newPage(): + self.current_page_num += 1 self.first_page = False mf.render(self.painter) nsl = evaljs('paged_display.next_screen_location()').toInt() @@ -297,11 +302,10 @@ class PDFWriter(QObject): # {{{ amap = self.bridge_value if not isinstance(amap, dict): amap = {} # Some javascript error occurred - pages = self.doc.page_count() - self.outline.set_pos(self.current_item, None, pages, 0) + self.outline.set_pos(self.current_item, None, start_page, 0) for anchor, x in amap.iteritems(): pagenum, ypos = x - self.outline.set_pos(self.current_item, anchor, pages + pagenum, ypos) + self.outline.set_pos(self.current_item, anchor, start_page + pagenum, ypos) def append_doc(self, outpath): doc = self.podofo.PDFDoc() From 4e20f776bcfc8ebd100a9ed1c4e8bb62148e49e2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 00:06:54 +0530 Subject: [PATCH 30/39] When vieweing a book on the device by clicking in the book details panel, first copy the book off the device, this allows it to work for the smart device driver and the MTP driver --- src/calibre/gui2/actions/view.py | 18 ++++++++++-------- src/calibre/gui2/book_details.py | 7 ++++--- src/calibre/gui2/init.py | 2 ++ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index 43e9dad5c4..5a7a991607 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -256,6 +256,15 @@ class ViewAction(InterfaceAction): db.prefs['gui_view_history'] = history[:vh] self.build_menus(db) + def view_device_book(self, path): + pt = PersistentTemporaryFile('_view_device_book'+\ + os.path.splitext(path)[1]) + self.persistent_files.append(pt) + pt.close() + self.gui.device_manager.view_book( + Dispatcher(self.book_downloaded_for_viewing), + path, pt.name) + def _view_books(self, rows): if not rows or len(rows) == 0: self._launch_viewer() @@ -270,12 +279,5 @@ class ViewAction(InterfaceAction): else: paths = self.gui.current_view().model().paths(rows) for path in paths: - pt = PersistentTemporaryFile('_viewer_'+\ - os.path.splitext(path)[1]) - self.persistent_files.append(pt) - pt.close() - self.gui.device_manager.view_book(\ - Dispatcher(self.book_downloaded_for_viewing), - path, pt.name) - + self.view_device_book(path) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index bf5fbe77bd..f03015f4ad 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -19,8 +19,8 @@ from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata.sources.identify import urls_from_identifiers from calibre.constants import filesystem_encoding from calibre.library.comments import comments_to_html -from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data, - gprefs, rating_font) +from calibre.gui2 import (config, open_url, pixmap_to_data, gprefs, + rating_font) from calibre.utils.icu import sort_key from calibre.utils.formatter import EvalFormatter from calibre.utils.date import is_date_undefined @@ -569,6 +569,7 @@ class BookDetails(QWidget): # {{{ files_dropped = pyqtSignal(object, object) cover_changed = pyqtSignal(object, object) cover_removed = pyqtSignal(object) + view_device_book = pyqtSignal(object) # Drag 'n drop {{{ DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS @@ -643,7 +644,7 @@ class BookDetails(QWidget): # {{{ id_, fmt = val.split(':') self.view_specific_format.emit(int(id_), fmt) elif typ == 'devpath': - open_local_file(val) + self.view_device_book.emit(val) else: try: open_url(QUrl(link, QUrl.TolerantMode)) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index a82dfec7fc..338a558f29 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -269,6 +269,8 @@ class LayoutMixin(object): # {{{ self.iactions['Remove Books'].remove_format_by_id) self.book_details.save_specific_format.connect( self.iactions['Save To Disk'].save_library_format_by_ids) + self.book_details.view_device_book.connect( + self.iactions['View'].view_device_book) m = self.library_view.model() if m.rowCount(None) > 0: From def42a1da369519eafbce574aa9958d6c7420701 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 00:44:18 +0530 Subject: [PATCH 31/39] PDF Output: Use less memory when writing out the PDF file --- src/calibre/ebooks/pdf/writer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index 92054e2f76..9ba9b5c20f 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -15,11 +15,12 @@ from PyQt4.Qt import (QEventLoop, QObject, QPrinter, QSizeF, Qt, QPainter, QPixmap, QTimer, pyqtProperty, QString, QSize) from PyQt4.QtWebKit import QWebView, QWebPage, QWebSettings +from calibre.constants import filesystem_encoding from calibre.ptempfile import PersistentTemporaryDirectory from calibre.ebooks.pdf.pageoptions import (unit, paper_size, orientation) from calibre.ebooks.pdf.outline_writer import Outline from calibre.ebooks.metadata import authors_to_string -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile from calibre import (__appname__, __version__, fit_image, isosx, force_unicode) from calibre.ebooks.oeb.display.webview import load_html @@ -350,8 +351,12 @@ class PDFWriter(QObject): # {{{ if self.metadata.tags: self.doc.keywords = self.metadata.tags self.outline(self.doc) - raw = self.doc.write() - self.out_stream.write(raw) + with TemporaryFile(u'pdf_out.pdf') as tf: + if isinstance(tf, unicode): + tf = tf.encode(filesystem_encoding) + self.doc.save(tf) + with open(tf, 'rb') as src: + shutil.copyfileobj(src, self.out_stream) self.render_succeeded = True finally: self._delete_tmpdir() From 0e6cce50e6ad70fffc4917d2bb32ab273a267077 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 09:12:20 +0530 Subject: [PATCH 32/39] Update Business Week Magazine --- recipes/bwmagazine2.recipe | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/recipes/bwmagazine2.recipe b/recipes/bwmagazine2.recipe index 300d71806a..77143bbefc 100644 --- a/recipes/bwmagazine2.recipe +++ b/recipes/bwmagazine2.recipe @@ -33,38 +33,36 @@ class BusinessWeekMagazine(BasicNewsRecipe): div0 = soup.find ('div', attrs={'class':'column left'}) section_title = '' feeds = OrderedDict() - articles = [] - for div in div0.findAll('a'): + for div in div0.findAll('h4'): + articles = [] section_title = self.tag_to_string(div.findPrevious('h3')).strip() - self.log('Processing section:', section_title) - title=self.tag_to_string(div).strip() - url=div['href'] + title=self.tag_to_string(div.a).strip() + url=div.a['href'] soup0 = self.index_to_soup(url) urlprint=soup0.find('li', attrs={'class':'print'}).a['href'] articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + div1 = soup.find ('div', attrs={'class':'column center'}) section_title = '' - articles = [] - for div in div1.findAll('a'): + for div in div1.findAll('h5'): + articles = [] desc=self.tag_to_string(div.findNext('p')).strip() section_title = self.tag_to_string(div.findPrevious('h3')).strip() - self.log('Processing section:', section_title) - title=self.tag_to_string(div).strip() - url=div['href'] + title=self.tag_to_string(div.a).strip() + url=div.a['href'] soup0 = self.index_to_soup(url) urlprint=soup0.find('li', attrs={'class':'print'}).a['href'] articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles ans = [(key, val) for key, val in feeds.iteritems()] return ans - From a907f919dab4b6d76a67fc6f9789710076161b64 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 09:33:54 +0530 Subject: [PATCH 33/39] When adding books to calibre and the book does not have a published date, set the published date to undefined rather than todays date --- src/calibre/library/database2.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index f1103f57ee..17c01a6f56 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -32,7 +32,7 @@ from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename, samefile from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp, - parse_only_date) + parse_only_date, UNDEFINED_DATE) from calibre.utils.config import prefs, tweaks, from_json, to_json from calibre.utils.icu import sort_key, strcmp, lower from calibre.utils.search_query_parser import saved_searches, set_saved_searches @@ -2498,16 +2498,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.notify('metadata', [id]) def set_pubdate(self, id, dt, notify=True, commit=True): - if dt: - if isinstance(dt, basestring): - dt = parse_only_date(dt) - self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) - self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) - self.dirtied([id], commit=False) - if commit: - self.conn.commit() - if notify: - self.notify('metadata', [id]) + if not dt: + dt = UNDEFINED_DATE + if isinstance(dt, basestring): + dt = parse_only_date(dt) + self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) + self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) + self.dirtied([id], commit=False) + if commit: + self.conn.commit() + if notify: + self.notify('metadata', [id]) def set_publisher(self, id, publisher, notify=True, commit=True, @@ -3344,7 +3345,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, ignore_errors=True, commit=True) if cover is not None: try: @@ -3386,7 +3387,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, commit=True, ignore_errors=True) npath = self.run_import_plugins(path, format) format = os.path.splitext(npath)[-1].lower().replace('.', '').upper() @@ -3426,7 +3427,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if mi.timestamp is None: mi.timestamp = utcnow() if mi.pubdate is None: - mi.pubdate = utcnow() + mi.pubdate = UNDEFINED_DATE self.set_metadata(id, mi, ignore_errors=True, commit=True) if preserve_uuid and mi.uuid: self.set_uuid(id, mi.uuid, commit=False) From 9360c5833a740336e58991c3d25b71d195e67335 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 09:35:50 +0530 Subject: [PATCH 34/39] Fix Chronicle of Higher Education --- recipes/chronicle_higher_ed.recipe | 32 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/recipes/chronicle_higher_ed.recipe b/recipes/chronicle_higher_ed.recipe index 7ed834a4e5..f0188d4d77 100644 --- a/recipes/chronicle_higher_ed.recipe +++ b/recipes/chronicle_higher_ed.recipe @@ -13,13 +13,13 @@ class Chronicle(BasicNewsRecipe): keep_only_tags = [ dict(name='div', attrs={'class':'article'}), ] - remove_tags = [dict(name='div',attrs={'class':'related module1'})] + remove_tags = [dict(name='div',attrs={'class':['related module1','maintitle']}), + dict(name='div', attrs={'id':['section-nav','icon-row']})] no_javascript = True no_stylesheets = True needs_subscription = True - def get_browser(self): br = BasicNewsRecipe.get_browser() if self.username is not None and self.password is not None: @@ -27,7 +27,7 @@ class Chronicle(BasicNewsRecipe): br.select_form(nr=1) br['username'] = self.username br['password'] = self.password - br.submit() + br.submit() return br def parse_index(self): @@ -47,33 +47,35 @@ class Chronicle(BasicNewsRecipe): #Go to the main body soup = self.index_to_soup(issueurl) - div0 = soup.find ('div', attrs={'id':'article-body'}) + div = soup.find ('div', attrs={'id':'article-body'}) feeds = OrderedDict() - for div in div0.findAll('div',attrs={'class':'module1'}): - section_title = self.tag_to_string(div.find('h3')) - for post in div.findAll('li',attrs={'class':'sub-promo'}): - articles = [] - a=post.find('a', href=True) + section_title = '' + for post in div.findAll('li'): + articles = [] + a=post.find('a', href=True) + if a is not None: title=self.tag_to_string(a) url="http://chronicle.com"+a['href'].strip() + sectiontitle=post.findPrevious('h3') + if sectiontitle is None: + sectiontitle=post.findPrevious('h4') + section_title=self.tag_to_string(sectiontitle) desc=self.tag_to_string(post.find('p')) articles.append({'title':title, 'url':url, 'description':desc, 'date':''}) - if articles: - if section_title not in feeds: - feeds[section_title] = [] - feeds[section_title] += articles + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles ans = [(key, val) for key, val in feeds.iteritems()] return ans def preprocess_html(self,soup): #process all the images for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}): - noscripts=div.find('noscript').a div.replaceWith(noscripts) for div0 in soup.findAll('div',text='Powered by Tableau'): div0.extract() return soup - From a4b16899014864d5b403c79a1ae25c0689ca1c8d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 09:37:42 +0530 Subject: [PATCH 35/39] Fix Financial Times (UK) --- recipes/financial_times_uk.recipe | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/recipes/financial_times_uk.recipe b/recipes/financial_times_uk.recipe index 4c331f115f..16295905bc 100644 --- a/recipes/financial_times_uk.recipe +++ b/recipes/financial_times_uk.recipe @@ -10,7 +10,7 @@ from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe class FinancialTimes(BasicNewsRecipe): - title = 'Financial Times - UK printed edition' + title = 'Financial Times (UK)' __author__ = 'Darko Miletic' description = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy." publisher = 'The Financial Times Ltd.' @@ -101,17 +101,19 @@ class FinancialTimes(BasicNewsRecipe): def parse_index(self): feeds = [] soup = self.index_to_soup(self.INDEX) + dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div')) + self.timefmt = ' [%s]'%dates wide = soup.find('div',attrs={'class':'wide'}) if not wide: return feeds strest = wide.findAll('h3', attrs={'class':'section'}) if not strest: return feeds - st = wide.find('h4',attrs={'class':'section-no-arrow'}) + st = wide.findAll('h4',attrs={'class':'section-no-arrow'}) if st: - strest.insert(0,st) + st.extend(strest) count = 0 - for item in strest: + for item in st: count = count + 1 if self.test and count > 2: return feeds @@ -151,7 +153,7 @@ class FinancialTimes(BasicNewsRecipe): def get_cover_url(self): cdate = datetime.date.today() if cdate.isoweekday() == 7: - cdate -= datetime.timedelta(days=1) + cdate -= datetime.timedelta(days=1) return cdate.strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf') def get_obfuscated_article(self, url): @@ -163,9 +165,8 @@ class FinancialTimes(BasicNewsRecipe): count = 10 except: print "Retrying download..." - count += 1 + count += 1 self.temp_files.append(PersistentTemporaryFile('_fa.html')) self.temp_files[-1].write(html) self.temp_files[-1].close() return self.temp_files[-1].name - \ No newline at end of file From b3fd7d3e0153b3aa668e9c473452c8f5cfd5c610 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 09:38:42 +0530 Subject: [PATCH 36/39] ... --- src/calibre/ebooks/pdf/writer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index 9ba9b5c20f..a547d3e650 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -122,7 +122,7 @@ class PDFMetadata(object): # {{{ self.author = force_unicode(self.author) # }}} -class Page(QWebPage): +class Page(QWebPage): # {{{ def __init__(self, opts, log): self.log = log @@ -153,6 +153,7 @@ class Page(QWebPage): def javaScriptAlert(self, frame, msg): self.log(unicode(msg)) +# }}} class PDFWriter(QObject): # {{{ From 5736706846d6bf68723c1a595516f22986ee95c7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 14:59:20 +0530 Subject: [PATCH 37/39] podofo: Implement writing to python file objects --- setup/extensions.py | 1 + src/calibre/ebooks/pdf/writer.py | 10 +- src/calibre/utils/podofo/__init__.py | 15 ++- src/calibre/utils/podofo/doc.cpp | 12 ++ src/calibre/utils/podofo/global.h | 1 + src/calibre/utils/podofo/output.cpp | 172 +++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 11 deletions(-) create mode 100644 src/calibre/utils/podofo/output.cpp diff --git a/setup/extensions.py b/setup/extensions.py index 2f2e2aa9ba..f4ed22687b 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -139,6 +139,7 @@ extensions = [ Extension('podofo', [ 'calibre/utils/podofo/utils.cpp', + 'calibre/utils/podofo/output.cpp', 'calibre/utils/podofo/doc.cpp', 'calibre/utils/podofo/outline.cpp', 'calibre/utils/podofo/podofo.cpp', diff --git a/src/calibre/ebooks/pdf/writer.py b/src/calibre/ebooks/pdf/writer.py index a547d3e650..a9cb951e35 100644 --- a/src/calibre/ebooks/pdf/writer.py +++ b/src/calibre/ebooks/pdf/writer.py @@ -15,12 +15,11 @@ from PyQt4.Qt import (QEventLoop, QObject, QPrinter, QSizeF, Qt, QPainter, QPixmap, QTimer, pyqtProperty, QString, QSize) from PyQt4.QtWebKit import QWebView, QWebPage, QWebSettings -from calibre.constants import filesystem_encoding from calibre.ptempfile import PersistentTemporaryDirectory from calibre.ebooks.pdf.pageoptions import (unit, paper_size, orientation) from calibre.ebooks.pdf.outline_writer import Outline from calibre.ebooks.metadata import authors_to_string -from calibre.ptempfile import PersistentTemporaryFile, TemporaryFile +from calibre.ptempfile import PersistentTemporaryFile from calibre import (__appname__, __version__, fit_image, isosx, force_unicode) from calibre.ebooks.oeb.display.webview import load_html @@ -352,12 +351,7 @@ class PDFWriter(QObject): # {{{ if self.metadata.tags: self.doc.keywords = self.metadata.tags self.outline(self.doc) - with TemporaryFile(u'pdf_out.pdf') as tf: - if isinstance(tf, unicode): - tf = tf.encode(filesystem_encoding) - self.doc.save(tf) - with open(tf, 'rb') as src: - shutil.copyfileobj(src, self.out_stream) + self.doc.save_to_fileobj(self.out_stream) self.render_succeeded = True finally: self._delete_tmpdir() diff --git a/src/calibre/utils/podofo/__init__.py b/src/calibre/utils/podofo/__init__.py index 3134dcd1ba..13c12a9bb3 100644 --- a/src/calibre/utils/podofo/__init__.py +++ b/src/calibre/utils/podofo/__init__.py @@ -94,9 +94,8 @@ def delete_all_but(path, pages): if page not in pages: p.delete_page(page) - raw = p.write() with open(path, 'wb') as f: - f.write(raw) + f.save_to_fileobj(path) def test_outline(src): podofo = get_podofo() @@ -114,7 +113,17 @@ def test_outline(src): f.write(raw) print 'Outlined PDF:', out +def test_save_to(src, dest): + podofo = get_podofo() + p = podofo.PDFDoc() + with open(src, 'rb') as f: + raw = f.read() + p.load(raw) + with open(dest, 'wb') as out: + p.save_to_fileobj(out) + print ('Wrote PDF of size:', out.tell()) + if __name__ == '__main__': import sys - test_outline(sys.argv[-1]) + test_save_to(sys.argv[-2], sys.argv[-1]) diff --git a/src/calibre/utils/podofo/doc.cpp b/src/calibre/utils/podofo/doc.cpp index 7166b2320e..90bfc27921 100644 --- a/src/calibre/utils/podofo/doc.cpp +++ b/src/calibre/utils/podofo/doc.cpp @@ -104,6 +104,15 @@ PDFDoc_write(PDFDoc *self, PyObject *args) { return ans; } + +static PyObject * +PDFDoc_save_to_fileobj(PDFDoc *self, PyObject *args) { + PyObject *f; + + if (!PyArg_ParseTuple(args, "O", &f)) return NULL; + return write_doc(self->doc, f); +} + // }}} // extract_first_page() {{{ @@ -453,6 +462,9 @@ static PyMethodDef PDFDoc_methods[] = { {"write", (PyCFunction)PDFDoc_write, METH_VARARGS, "Return the PDF document as a bytestring." }, + {"save_to_fileobj", (PyCFunction)PDFDoc_save_to_fileobj, METH_VARARGS, + "Write the PDF document to the soecified file-like object." + }, {"extract_first_page", (PyCFunction)PDFDoc_extract_first_page, METH_VARARGS, "extract_first_page() -> Remove all but the first page." }, diff --git a/src/calibre/utils/podofo/global.h b/src/calibre/utils/podofo/global.h index fa9a141b21..4a180d86a0 100644 --- a/src/calibre/utils/podofo/global.h +++ b/src/calibre/utils/podofo/global.h @@ -41,6 +41,7 @@ extern void podofo_set_exception(const PdfError &err); extern PyObject * podofo_convert_pdfstring(const PdfString &s); extern PdfString * podofo_convert_pystring(PyObject *py); extern PdfString * podofo_convert_pystring_single_byte(PyObject *py); +extern PyObject* write_doc(PdfMemDocument *doc, PyObject *f); } diff --git a/src/calibre/utils/podofo/output.cpp b/src/calibre/utils/podofo/output.cpp new file mode 100644 index 0000000000..63e1270af6 --- /dev/null +++ b/src/calibre/utils/podofo/output.cpp @@ -0,0 +1,172 @@ +/* + * output.cpp + * Copyright (C) 2012 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include "global.h" + +using namespace PoDoFo; + +class pyerr : public std::exception { +}; + +class OutputDevice : public PdfOutputDevice { + + private: + PyObject *file; + size_t written; + + void update_written() { + size_t pos; + pos = Tell(); + if (pos > written) written = pos; + } + + public: + OutputDevice(PyObject *f) : file(f), written(0) { Py_XINCREF(file); } + ~OutputDevice() { Py_XDECREF(file); file = NULL; } + + size_t GetLength() const { return written; } + + long PrintVLen(const char* pszFormat, va_list args) { + char buf[10]; + int res; + + if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); } + + res = PyOS_vsnprintf(buf, 1, pszFormat, args); + if (res < 0) { + PyErr_SetString(PyExc_Exception, "Something bad happend while calling PyOS_vsnprintf"); + throw pyerr(); + } + return static_cast(res+1); + } + + void PrintV( const char* pszFormat, long lBytes, va_list args ) { + char *buf; + int res; + + if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); } + + buf = new (std::nothrow) char[lBytes+1]; + if (buf == NULL) { PyErr_NoMemory(); throw pyerr(); } + + res = PyOS_vsnprintf(buf, lBytes, pszFormat, args); + + if (res < 0) { + PyErr_SetString(PyExc_Exception, "Something bad happend while calling PyOS_vsnprintf"); + delete[] buf; + throw pyerr(); + } + + Write(buf, static_cast(res)); + delete[] buf; + } + + void Print( const char* pszFormat, ... ) + { + va_list args; + long lBytes; + + va_start( args, pszFormat ); + lBytes = PrintVLen(pszFormat, args); + va_end( args ); + + va_start( args, pszFormat ); + PrintV(pszFormat, lBytes, args); + va_end( args ); + } + + size_t Read( char* pBuffer, size_t lLen ) { + PyObject *ret; + char *buf = NULL; + Py_ssize_t len = 0; + + ret = PyObject_CallMethod(file, (char*)"read", (char*)"n", static_cast(lLen)); + if (ret != NULL) { + if (PyBytes_AsStringAndSize(ret, &buf, &len) != -1) { + memcpy(pBuffer, buf, len); + Py_DECREF(ret); + return static_cast(len); + } + Py_DECREF(ret); + } + + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to read data from python file object"); + + throw pyerr(); + + } + + void Seek(size_t offset) { + PyObject *ret; + ret = PyObject_CallMethod(file, (char*)"seek", (char*)"n", static_cast(offset)); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to seek in python file object"); + throw pyerr(); + } + Py_DECREF(ret); + } + + size_t Tell() const { + PyObject *ret; + unsigned long ans; + + ret = PyObject_CallMethod(file, (char*)"tell", NULL); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to call tell() on python file object"); + throw pyerr(); + } + if (!PyNumber_Check(ret)) { + Py_DECREF(ret); + PyErr_SetString(PyExc_Exception, "tell() method did not return a number"); + throw pyerr(); + } + ans = PyInt_AsUnsignedLongMask(ret); + Py_DECREF(ret); + if (PyErr_Occurred() != NULL) throw pyerr(); + + return static_cast(ans); + } + + void Write(const char* pBuffer, size_t lLen) { + PyObject *ret; + + ret = PyObject_CallMethod(file, (char*)"write", (char*)"s#", pBuffer, (int)lLen); + if (ret == NULL) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "Failed to call write() on python file object"); + throw pyerr(); + } + Py_DECREF(ret); + update_written(); + } + + void Flush() { + Py_XDECREF(PyObject_CallMethod(file, (char*)"flush", NULL)); + } + +}; + + +PyObject* pdf::write_doc(PdfMemDocument *doc, PyObject *f) { + OutputDevice d(f); + + try { + doc->Write(&d); + } catch(const PdfError & err) { + podofo_set_exception(err); return NULL; + } catch (...) { + if (PyErr_Occurred() == NULL) + PyErr_SetString(PyExc_Exception, "An unknown error occurred while trying to write the pdf to the file object"); + return NULL; + } + + Py_RETURN_NONE; +} + From efac50df463a5081dac925023356c31b6880eaf2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 15:13:12 +0530 Subject: [PATCH 38/39] PDF Output: Do not error out when generating an outline which points to pages that have been removed. Fixes #1044799 (Private bug) --- src/calibre/ebooks/pdf/outline_writer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/ebooks/pdf/outline_writer.py b/src/calibre/ebooks/pdf/outline_writer.py index c89f2d9f41..64d11f0208 100644 --- a/src/calibre/ebooks/pdf/outline_writer.py +++ b/src/calibre/ebooks/pdf/outline_writer.py @@ -47,14 +47,19 @@ class Outline(object): for child in toc: page, ypos = self.get_pos(child) text = child.text or _('Page %d')%page + if page >= self.page_count: + page = self.page_count - 1 cn = parent.create(text, page, True) self.add_children(child, cn) def __call__(self, doc): self.pos_map = dict(self.pos_map) + self.page_count = doc.page_count() for child in self.toc: page, ypos = self.get_pos(child) text = child.text or _('Page %d')%page + if page >= self.page_count: + page = self.page_count - 1 node = doc.create_outline(text, page) self.add_children(child, node) From f280dd4de72792658001d2481a64d96370d0cbd7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 2 Sep 2012 17:40:15 +0530 Subject: [PATCH 39/39] Remove PyOS_vsnprintf as it is broken on windows --- src/calibre/utils/podofo/output.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/calibre/utils/podofo/output.cpp b/src/calibre/utils/podofo/output.cpp index 63e1270af6..b0620f7f82 100644 --- a/src/calibre/utils/podofo/output.cpp +++ b/src/calibre/utils/podofo/output.cpp @@ -31,17 +31,21 @@ class OutputDevice : public PdfOutputDevice { size_t GetLength() const { return written; } long PrintVLen(const char* pszFormat, va_list args) { - char buf[10]; - int res; if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); } - res = PyOS_vsnprintf(buf, 1, pszFormat, args); +#ifdef _MSC_VER + return _vscprintf(pszFormat, args); +#else + char buf[10]; + int res; + res = vsnprintf(buf, 1, pszFormat, args); if (res < 0) { - PyErr_SetString(PyExc_Exception, "Something bad happend while calling PyOS_vsnprintf"); + PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf to get buffer length"); throw pyerr(); } return static_cast(res+1); +#endif } void PrintV( const char* pszFormat, long lBytes, va_list args ) { @@ -53,10 +57,11 @@ class OutputDevice : public PdfOutputDevice { buf = new (std::nothrow) char[lBytes+1]; if (buf == NULL) { PyErr_NoMemory(); throw pyerr(); } - res = PyOS_vsnprintf(buf, lBytes, pszFormat, args); + // Note: PyOS_vsnprintf produces broken output on windows + res = vsnprintf(buf, lBytes, pszFormat, args); if (res < 0) { - PyErr_SetString(PyExc_Exception, "Something bad happend while calling PyOS_vsnprintf"); + PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf"); delete[] buf; throw pyerr(); }