0.8.67+

recipes/bwmagazine2.recipe

@@ -33,12 +33,11 @@ class BusinessWeekMagazine(BasicNewsRecipe):
         div0 = soup.find ('div', attrs={'class':'column left'})
         section_title = ''
         feeds = OrderedDict()
+        for div in div0.findAll('h4'):
             articles = []
-        for div in div0.findAll('a'):
             section_title = self.tag_to_string(div.findPrevious('h3')).strip()
-            self.log('Processing section:', section_title)
+            title=self.tag_to_string(div.a).strip()
-            title=self.tag_to_string(div).strip()
+            url=div.a['href']
-            url=div['href']
             soup0 = self.index_to_soup(url)
             urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
             articles.append({'title':title, 'url':urlprint, 'description':'', 'date':''})
@@ -48,15 +47,15 @@ class BusinessWeekMagazine(BasicNewsRecipe):
                 if section_title not in feeds:
                     feeds[section_title] = []
                 feeds[section_title] += articles
+
         div1 = soup.find ('div', attrs={'class':'column center'})
         section_title = ''
+        for div in div1.findAll('h5'):
             articles = []
-        for div in div1.findAll('a'):
             desc=self.tag_to_string(div.findNext('p')).strip()
             section_title = self.tag_to_string(div.findPrevious('h3')).strip()
-            self.log('Processing section:', section_title)
+            title=self.tag_to_string(div.a).strip()
-            title=self.tag_to_string(div).strip()
+            url=div.a['href']
-            url=div['href']
             soup0 = self.index_to_soup(url)
             urlprint=soup0.find('li', attrs={'class':'print'}).a['href']
             articles.append({'title':title, 'url':urlprint, 'description':desc, 'date':''})
@@ -67,4 +66,3 @@ class BusinessWeekMagazine(BasicNewsRecipe):
                 feeds[section_title] += articles
         ans = [(key, val) for key, val in feeds.iteritems()]
         return ans
-

recipes/chronicle_higher_ed.recipe

@@ -13,13 +13,13 @@ class Chronicle(BasicNewsRecipe):
     keep_only_tags = [
             dict(name='div', attrs={'class':'article'}),
             ]
-    remove_tags = [dict(name='div',attrs={'class':'related module1'})]
+    remove_tags = [dict(name='div',attrs={'class':['related module1','maintitle']}),
+            dict(name='div', attrs={'id':['section-nav','icon-row']})]
     no_javascript = True
     no_stylesheets = True
 
 
     needs_subscription = True
-
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
         if self.username is not None and self.password is not None:
@@ -47,16 +47,20 @@ class Chronicle(BasicNewsRecipe):
 
         #Go to the main body
         soup = self.index_to_soup(issueurl)
-        div0 = soup.find ('div', attrs={'id':'article-body'})
+        div = soup.find ('div', attrs={'id':'article-body'})
 
         feeds = OrderedDict()
-        for div in div0.findAll('div',attrs={'class':'module1'}):
+        section_title = ''
-            section_title = self.tag_to_string(div.find('h3'))
+        for post in div.findAll('li'):
-            for post in div.findAll('li',attrs={'class':'sub-promo'}):
             articles = []
             a=post.find('a', href=True)
+            if a is not None:
                 title=self.tag_to_string(a)
                 url="http://chronicle.com"+a['href'].strip()
+                sectiontitle=post.findPrevious('h3')
+                if sectiontitle is None:
+                    sectiontitle=post.findPrevious('h4')
+                section_title=self.tag_to_string(sectiontitle)
                 desc=self.tag_to_string(post.find('p'))
                 articles.append({'title':title, 'url':url, 'description':desc, 'date':''})
 
@@ -70,10 +74,8 @@ class Chronicle(BasicNewsRecipe):
     def preprocess_html(self,soup):
         #process all the images
         for div in soup.findAll('div', attrs={'class':'tableauPlaceholder'}):
-
             noscripts=div.find('noscript').a
             div.replaceWith(noscripts)
         for div0 in soup.findAll('div',text='Powered by Tableau'):
             div0.extract()
         return soup
-

recipes/financial_times_uk.recipe

@@ -10,7 +10,7 @@ from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class FinancialTimes(BasicNewsRecipe):
-    title                 = 'Financial Times - UK printed edition'
+    title                 = 'Financial Times (UK)'
     __author__            = 'Darko Miletic'
     description           = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
     publisher             = 'The Financial Times Ltd.'
@@ -101,17 +101,19 @@ class FinancialTimes(BasicNewsRecipe):
     def parse_index(self):
         feeds = []
         soup = self.index_to_soup(self.INDEX)
+        dates= self.tag_to_string(soup.find('div', attrs={'class':'btm-links'}).find('div'))
+        self.timefmt = ' [%s]'%dates
         wide = soup.find('div',attrs={'class':'wide'})
         if not wide:
            return feeds
         strest = wide.findAll('h3', attrs={'class':'section'})
         if not strest:
            return feeds
-        st = wide.find('h4',attrs={'class':'section-no-arrow'})
+        st = wide.findAll('h4',attrs={'class':'section-no-arrow'})
         if st:
-           strest.insert(0,st)
+           st.extend(strest)
         count = 0
-        for item in strest:
+        for item in st:
             count = count + 1
             if self.test and count > 2:
                return feeds
@@ -168,4 +170,3 @@ class FinancialTimes(BasicNewsRecipe):
         self.temp_files[-1].write(html)
         self.temp_files[-1].close()
         return self.temp_files[-1].name
-        

setup/extensions.py

@@ -139,6 +139,7 @@ extensions = [
     Extension('podofo',
                     [
                         'calibre/utils/podofo/utils.cpp',
+                        'calibre/utils/podofo/output.cpp',
                         'calibre/utils/podofo/doc.cpp',
                         'calibre/utils/podofo/outline.cpp',
                         'calibre/utils/podofo/podofo.cpp',

src/calibre/customize/builtins.py

@@ -675,7 +675,6 @@ from calibre.devices.bambook.driver import BAMBOOK
 from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX
 from calibre.devices.smart_device_app.driver import SMART_DEVICE_APP
 
-
 # Order here matters. The first matched device is the one used.
 plugins += [
     HANLINV3,
@@ -749,6 +748,12 @@ plugins += [
     SMART_DEVICE_APP,
     USER_DEFINED,
 ]
+
+from calibre.utils.config_base import tweaks
+if tweaks.get('test_mtp_driver', False):
+    from calibre.devices.mtp.driver import MTP_DEVICE
+    plugins.append(MTP_DEVICE)
+
 # }}}
 
 # New metadata download plugins {{{

src/calibre/devices/__init__.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Device drivers.
 '''
 
-import sys, time, pprint, operator
+import sys, time, pprint, operator, re, os
 from functools import partial
 from StringIO import StringIO
 
@@ -27,6 +27,112 @@ def strftime(epoch, zone=time.gmtime):
     src[2] = INVERSE_MONTH_MAP[int(src[2])]
     return ' '.join(src)
 
+def build_template_regexp(template):
+    from calibre import prints
+
+    def replfunc(match, seen=None):
+        v = match.group(1)
+        if v in ['authors', 'author_sort']:
+            v = 'author'
+        if v in ('title', 'series', 'series_index', 'isbn', 'author'):
+            if v not in seen:
+                seen.add(v)
+                return '(?P<' + v + '>.+?)'
+        return '(.+?)'
+    s = set()
+    f = partial(replfunc, seen=s)
+
+    try:
+        template = template.rpartition('/')[2]
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+    except:
+        prints(u'Failed to parse template: %r'%template)
+        template = u'{title} - {authors}'
+        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+
+def create_upload_path(mdata, fname, template, sanitize,
+        prefix_path='',
+        path_type=os.path,
+        maxlen=250,
+        use_subdirs=True,
+        news_in_folder=True,
+        filename_callback=lambda x, y:x,
+        sanitize_path_components=lambda x: x
+        ):
+    from calibre.library.save_to_disk import get_components, config
+    from calibre.utils.filenames import shorten_components_to
+
+    special_tag = None
+    if mdata.tags:
+        for t in mdata.tags:
+            if t.startswith(_('News')) or t.startswith('/'):
+                special_tag = t
+                break
+
+    if mdata.tags and _('News') in mdata.tags:
+        try:
+            p = mdata.pubdate
+            date  = (p.year, p.month, p.day)
+        except:
+            today = time.localtime()
+            date = (today[0], today[1], today[2])
+        template = u"{title}_%d-%d-%d" % date
+
+    fname = sanitize(fname)
+    ext = path_type.splitext(fname)[1]
+
+    opts = config().parse()
+    if not isinstance(template, unicode):
+        template = template.decode('utf-8')
+    app_id = str(getattr(mdata, 'application_id', ''))
+    id_ = mdata.get('id', fname)
+    extra_components = get_components(template, mdata, id_,
+            timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
+    if not extra_components:
+        extra_components.append(sanitize(filename_callback(fname,
+            mdata)))
+    else:
+        extra_components[-1] = sanitize(filename_callback(extra_components[-1]+ext, mdata))
+
+    if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
+        extra_components[-1] = 'x' + extra_components[-1][1:]
+
+    if special_tag is not None:
+        name = extra_components[-1]
+        extra_components = []
+        tag = special_tag
+        if tag.startswith(_('News')):
+            if news_in_folder:
+                extra_components.append('News')
+        else:
+            for c in tag.split('/'):
+                c = sanitize(c)
+                if not c: continue
+                extra_components.append(c)
+        extra_components.append(name)
+
+    if not use_subdirs:
+        extra_components = extra_components[-1:]
+
+    def remove_trailing_periods(x):
+        ans = x
+        while ans.endswith('.'):
+            ans = ans[:-1].strip()
+        if not ans:
+            ans = 'x'
+        return ans
+
+    extra_components = list(map(remove_trailing_periods, extra_components))
+    components = shorten_components_to(maxlen - len(prefix_path), extra_components)
+    components = sanitize_path_components(components)
+    if prefix_path:
+        filepath = path_type.join(prefix_path, *components)
+    else:
+        filepath = path_type.join(*components)
+
+    return filepath
+
+
 def get_connected_device():
     from calibre.customize.ui import device_plugins
     from calibre.devices.scanner import DeviceScanner
@@ -115,8 +221,19 @@ def debug(ioreg_to_tmp=False, buf=None, plugins=None):
         out('Available plugins:', textwrap.fill(' '.join([x.__class__.__name__ for x in
             devplugins])))
         out(' ')
+        found_dev = False
+        for dev in devplugins:
+            if not dev.MANAGES_DEVICE_PRESENCE: continue
+            out('Looking for devices of type:', dev.__class__.__name__)
+            if dev.debug_managed_device_detection(s.devices, buf):
+                found_dev = True
+                break
+            out(' ')
+
+        if not found_dev:
             out('Looking for devices...')
             for dev in devplugins:
+                if dev.MANAGES_DEVICE_PRESENCE: continue
                 connected, det = s.is_device_connected(dev, debug=True)
                 if connected:
                     out('\t\tDetected possible device', dev.__class__.__name__)

src/calibre/devices/android/driver.py

@@ -197,7 +197,8 @@ class ANDROID(USBMS):
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
             'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
             'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
-            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0']
+            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0',
+            'COBY_MID']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -216,7 +217,7 @@ class ANDROID(USBMS):
             'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
             'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
             'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
-            'S5830I_CARD', 'MID7042', 'LINK-CREATE']
+            'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -226,7 +227,7 @@ class ANDROID(USBMS):
             'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
             'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
             'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
-            'USB_FLASH_DRIVER', 'ANDROID', 'MID7042']
+            'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/devices/cli.py

@@ -9,7 +9,7 @@ For usage information run the script.
 import StringIO, sys, time, os
 from optparse import OptionParser
 
-from calibre import __version__, __appname__
+from calibre import __version__, __appname__, human_readable
 from calibre.devices.errors import PathError
 from calibre.utils.terminfo import TerminalController
 from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked
@@ -18,16 +18,6 @@ from calibre.devices.scanner import DeviceScanner
 
 MINIMUM_COL_WIDTH = 12 #: Minimum width of columns in ls output
 
-def human_readable(size):
-    """ Convert a size in bytes into a human readle form """
-    if size < 1024: divisor, suffix = 1, ""
-    elif size < 1024*1024: divisor, suffix = 1024., "K"
-    elif size < 1024*1024*1024: divisor, suffix = 1024*1024, "M"
-    elif size < 1024*1024*1024*1024: divisor, suffix = 1024*1024, "G"
-    size = str(size/divisor)
-    if size.find(".") > -1: size = size[:size.find(".")+2]
-    return size + suffix
-
 class FileFormatter(object):
     def __init__(self, file, term):
         self.term = term
@@ -207,11 +197,19 @@ def main():
     scanner = DeviceScanner()
     scanner.scan()
     connected_devices = []
+
     for d in device_plugins():
         try:
             d.startup()
         except:
             print ('Startup failed for device plugin: %s'%d)
+        if d.MANAGES_DEVICE_PRESENCE:
+            cd = d.detect_managed_devices(scanner.devices)
+            if cd is not None:
+                connected_devices.append((cd, d))
+                dev = d
+                break
+            continue
         ok, det = scanner.is_device_connected(d)
         if ok:
             dev = d

src/calibre/devices/interface.py

@@ -81,6 +81,19 @@ class DevicePlugin(Plugin):
     #: by.
     NUKE_COMMENTS = None
 
+    #: If True indicates that  this driver completely manages device detection,
+    #: ejecting and so forth. If you set this to True, you *must* implement the
+    #: detect_managed_devices and debug_managed_device_detection methods.
+    #: A driver with this set to true is responsible for detection of devices,
+    #: managing a blacklist of devices, a list of ejected devices and so forth.
+    #: calibre will periodically call the detect_managed_devices() method and
+    #: is it returns a detected device, calibre will call open(). open() will
+    #: be called every time a device is returned even is previous calls to open()
+    #: failed, therefore the driver must maintain its own blacklist of failed
+    #: devices. Similarly, when ejecting, calibre will call eject() and then
+    #: assuming the next call to detect_managed_devices() returns None, it will
+    #: call post_yank_cleanup().
+    MANAGES_DEVICE_PRESENCE = False
 
     @classmethod
     def get_gui_name(cls):
@@ -196,6 +209,37 @@ class DevicePlugin(Plugin):
                                     return True, dev
         return False, None
 
+    def detect_managed_devices(self, devices_on_system, force_refresh=False):
+        '''
+        Called only if MANAGES_DEVICE_PRESENCE is True.
+
+        Scan for devices that this driver can handle. Should return a device
+        object if a device is found. This object will be passed to the open()
+        method as the connected_device. If no device is found, return None.
+
+        This method is called periodically by the GUI, so make sure it is not
+        too resource intensive. Use a cache to avoid repeatedly scanning the
+        system.
+
+        :param devices_on_system: Set of USB devices found on the system.
+
+        :param force_refresh: If True and the driver uses a cache to prevent
+        repeated scanning, the cache must be flushed.
+        '''
+        raise NotImplementedError()
+
+    def debug_managed_device_detection(self, devices_on_system, output):
+        '''
+        Called only if MANAGES_DEVICE_PRESENCE is True.
+
+        Should write information about the devices detected on the system to
+        output, which is a file like object.
+
+        Should return True if a device was detected and successfully opened,
+        otherwise False.
+        '''
+        raise NotImplementedError()
+
     # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
@@ -270,6 +314,9 @@ class DevicePlugin(Plugin):
         '''
         Un-mount / eject the device from the OS. This does not check if there
         are pending GUI jobs that need to communicate with the device.
+
+        NOTE: That this method may not be called on the same thread as the rest
+        of the device methods.
         '''
         raise NotImplementedError()
 
@@ -496,6 +543,10 @@ class DevicePlugin(Plugin):
         '''
         Given a list of paths, returns another list of paths. These paths
         point to addable versions of the books.
+
+        If there is an error preparing a book, then instead of a path, the
+        position in the returned list for that book should be a three tuple:
+            (original_path, the exception instance, traceback)
         '''
         return paths
 

src/calibre/devices/mtp/base.py

@@ -9,8 +9,14 @@ __docformat__ = 'restructuredtext en'
 
 from functools import wraps
 
+from calibre import prints
+from calibre.constants import DEBUG
 from calibre.devices.interface import DevicePlugin
 
+def debug(*args, **kwargs):
+    if DEBUG:
+        prints('MTP:', *args, **kwargs)
+
 def synchronous(func):
     @wraps(func)
     def synchronizer(self, *args, **kwargs):
@@ -26,11 +32,6 @@ class MTPDeviceBase(DevicePlugin):
     author = 'Kovid Goyal'
     version = (1, 0, 0)
 
-    THUMBNAIL_HEIGHT = 128
-    CAN_SET_METADATA = []
-
-    BACKLOADING_ERROR_MESSAGE = None
-
     def __init__(self, *args, **kwargs):
         DevicePlugin.__init__(self, *args, **kwargs)
         self.progress_reporter = None
@@ -53,4 +54,17 @@ class MTPDeviceBase(DevicePlugin):
         # return False
         return False
 
+    def build_template_regexp(self):
+        from calibre.devices import build_template_regexp
+        return build_template_regexp(self.save_template)
+
+    @property
+    def default_save_template(cls):
+        from calibre.library.save_to_disk import config
+        return config().parse().send_template
+
+    @property
+    def save_template(self):
+        # TODO: Use the device specific template here
+        return self.default_save_template
 

src/calibre/devices/mtp/books.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+from calibre.devices.interface import BookList as BL
+from calibre.ebooks.metadata.book.base import Metadata
+from calibre.ebooks.metadata.book.json_codec import JsonCodec
+from calibre.utils.date import utcnow
+
+class BookList(BL):
+
+    def __init__(self, storage_id):
+        self.storage_id = storage_id
+
+    def supports_collections(self):
+        return False
+
+class Book(Metadata):
+
+    def __init__(self, storage_id, lpath, other=None):
+        Metadata.__init__(self, _('Unknown'), other=other)
+        self.storage_id, self.lpath = storage_id, lpath
+        self.lpath = self.path = self.lpath.replace(os.sep, '/')
+        self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')])
+        self.datetime = utcnow().timetuple()
+        self.thumbail = None
+
+    def matches_file(self, mtp_file):
+        return (self.storage_id == mtp_file.storage_id and
+                self.mtp_relpath == mtp_file.mtp_relpath)
+
+class JSONCodec(JsonCodec):
+    pass
+

src/calibre/devices/mtp/driver.py

@@ -7,31 +7,43 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import json, pprint
+import json, traceback, posixpath, importlib, os
 from io import BytesIO
 
+from calibre import prints
 from calibre.constants import iswindows, numeric_version
+from calibre.devices.mtp.base import debug
+from calibre.ptempfile import SpooledTemporaryFile, PersistentTemporaryDirectory
 from calibre.utils.config import from_json, to_json
 from calibre.utils.date import now, isoformat
 
-if iswindows:
+BASE = importlib.import_module('calibre.devices.mtp.%s.driver'%(
-    from calibre.devices.mtp.windows.driver import MTP_DEVICE as BASE
+    'windows' if iswindows else 'unix')).MTP_DEVICE
-    BASE
-else:
-    from calibre.devices.mtp.unix.driver import MTP_DEVICE as BASE
-pprint
 
 class MTP_DEVICE(BASE):
 
     METADATA_CACHE = 'metadata.calibre'
     DRIVEINFO = 'driveinfo.calibre'
+    CAN_SET_METADATA = []
+    NEWS_IN_FOLDER = True
+    MAX_PATH_LEN = 230
+    THUMBNAIL_HEIGHT = 160
+    THUMBNAIL_WIDTH = 120
+    CAN_SET_METADATA = []
+    BACKLOADING_ERROR_MESSAGE = None
+    MANAGES_DEVICE_PRESENCE = True
 
+    def open(self, devices, library_uuid):
+        self.current_library_uuid = library_uuid
+        BASE.open(self, devices, library_uuid)
+
+    # Device information {{{
     def _update_drive_info(self, storage, location_code, name=None):
         import uuid
         f = storage.find_path((self.DRIVEINFO,))
         dinfo = {}
         if f is not None:
-            stream = self.get_file(f)
+            stream = self.get_mtp_file(f)
             try:
                 dinfo = json.load(stream, object_hook=from_json)
             except:
@@ -51,10 +63,6 @@ class MTP_DEVICE(BASE):
         self.put_file(storage, self.DRIVEINFO, BytesIO(raw), len(raw))
         self.driveinfo = dinfo
 
-    def open(self, devices, library_uuid):
-        self.current_library_uuid = library_uuid
-        BASE.open(self, devices, library_uuid)
-
     def get_device_information(self, end_session=True):
         self.report_progress(1.0, _('Get device information...'))
         self.driveinfo = {}
@@ -80,6 +88,154 @@ class MTP_DEVICE(BASE):
             return
         self._update_drive_info(self.filesystem_cache.storage(sid),
                 location_code, name=name)
+    # }}}
+
+    # Get list of books from device, with metadata {{{
+    def books(self, oncard=None, end_session=True):
+        from calibre.devices.mtp.books import JSONCodec
+        from calibre.devices.mtp.books import BookList, Book
+        sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
+                self._main_id)
+        if sid is None:
+            return BookList(None)
+
+        bl = BookList(sid)
+        # If True then there is a mismatch between the ebooks on the device and
+        # the metadata cache
+        need_sync = False
+        all_books = list(self.filesystem_cache.iterebooks(sid))
+        steps = len(all_books) + 2
+        count = 0
+
+        self.report_progress(0, _('Reading metadata from device'))
+        # Read the cache if it exists
+        storage = self.filesystem_cache.storage(sid)
+        cache = storage.find_path((self.METADATA_CACHE,))
+        if cache is not None:
+            json_codec = JSONCodec()
+            try:
+                stream = self.get_mtp_file(cache)
+                json_codec.decode_from_file(stream, bl, Book, sid)
+            except:
+                need_sync = True
+
+        relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
+
+        for mtp_file in all_books:
+            count += 1
+            relpath = mtp_file.mtp_relpath
+            idx = relpath_cache.get(relpath, None)
+            if idx is not None:
+                cached_metadata = bl[idx]
+                del relpath_cache[relpath]
+                if cached_metadata.size == mtp_file.size:
+                    cached_metadata.datetime = mtp_file.last_modified.timetuple()
+                    cached_metadata.path = mtp_file.mtp_id_path
+                    debug('Using cached metadata for',
+                            '/'.join(mtp_file.full_path))
+                    continue # No need to update metadata
+                book = cached_metadata
+            else:
+                book = Book(sid, '/'.join(relpath))
+                bl.append(book)
+
+            need_sync = True
+            self.report_progress(count/steps, _('Reading metadata from %s')%
+                    ('/'.join(relpath)))
+            try:
+                book.smart_update(self.read_file_metadata(mtp_file))
+                debug('Read metadata for', '/'.join(mtp_file.full_path))
+            except:
+                prints('Failed to read metadata from',
+                        '/'.join(mtp_file.full_path))
+                traceback.print_exc()
+            book.size = mtp_file.size
+            book.datetime = mtp_file.last_modified.timetuple()
+            book.path = mtp_file.mtp_id_path
+
+        # Remove books in the cache that no longer exist
+        for idx in sorted(relpath_cache.itervalues(), reverse=True):
+            del bl[idx]
+            need_sync = True
+
+        if need_sync:
+            self.report_progress(count/steps, _('Updating metadata cache on device'))
+            self.write_metadata_cache(storage, bl)
+        self.report_progress(1, _('Finished reading metadata from device'))
+        return bl
+
+    def read_file_metadata(self, mtp_file):
+        from calibre.ebooks.metadata.meta import get_metadata
+        from calibre.customize.ui import quick_metadata
+        ext = mtp_file.name.rpartition('.')[-1].lower()
+        stream = self.get_mtp_file(mtp_file)
+        with quick_metadata:
+            return get_metadata(stream, stream_type=ext,
+                    force_read_metadata=True,
+                    pattern=self.build_template_regexp())
+
+    def write_metadata_cache(self, storage, bl):
+        from calibre.devices.mtp.books import JSONCodec
+
+        if bl.storage_id != storage.storage_id:
+            # Just a sanity check, should never happen
+            return
+
+        json_codec = JSONCodec()
+        stream = SpooledTemporaryFile(10*(1024**2))
+        json_codec.encode_to_file(stream, bl)
+        size = stream.tell()
+        stream.seek(0)
+        self.put_file(storage, self.METADATA_CACHE, stream, size)
+
+    def sync_booklists(self, booklists, end_session=True):
+        for bl in booklists:
+            if getattr(bl, 'storage_id', None) is None:
+                continue
+            storage = self.filesystem_cache.storage(bl.storage_id)
+            if storage is None:
+                continue
+            self.write_metadata_cache(storage, bl)
+
+    # }}}
+
+    # Get files from the device {{{
+    def get_file(self, path, outfile, end_session=True):
+        f = self.filesystem_cache.resolve_mtp_id_path(path)
+        self.get_mtp_file(f, outfile)
+
+    def prepare_addable_books(self, paths):
+        tdir = PersistentTemporaryDirectory('_prepare_mtp')
+        ans = []
+        for path in paths:
+            try:
+                f = self.filesystem_cache.resolve_mtp_id_path(path)
+            except Exception as e:
+                ans.append((path, e, traceback.format_exc()))
+                continue
+            base = os.path.join(tdir, '%s'%f.object_id)
+            os.mkdir(base)
+            with open(os.path.join(base, f.name), 'wb') as out:
+                try:
+                    self.get_mtp_file(f, out)
+                except Exception as e:
+                    ans.append((path, e, traceback.format_exc()))
+                else:
+                    ans.append(out.name)
+        return ans
+    # }}}
+
+    def create_upload_path(self, path, mdata, fname):
+        from calibre.devices import create_upload_path
+        from calibre.utils.filenames import ascii_filename as sanitize
+        filepath = create_upload_path(mdata, fname, self.save_template, sanitize,
+                prefix_path=path,
+                path_type=posixpath,
+                maxlen=self.MAX_PATH_LEN,
+                use_subdirs = True,
+                news_in_folder = self.NEWS_IN_FOLDER,
+                )
+        return tuple(x.lower() for x in filepath.split('/'))
 
 if __name__ == '__main__':
     dev = MTP_DEVICE(None)
@@ -92,8 +248,9 @@ if __name__ == '__main__':
         cd = dev.detect_managed_devices(devs)
         if cd is None:
             raise ValueError('Failed to detect MTP device')
+        dev.set_progress_reporter(prints)
         dev.open(cd, None)
-        pprint.pprint(dev.get_device_information())
+        dev.books()
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -7,17 +7,24 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import weakref, sys
+import weakref, sys, json
 from collections import deque
 from operator import attrgetter
 from future_builtins import map
+from datetime import datetime
 
 from calibre import human_readable, prints, force_unicode
+from calibre.utils.date import local_tz, as_utc
 from calibre.utils.icu import sort_key, lower
+from calibre.ebooks import BOOK_EXTENSIONS
+
+bexts = frozenset(BOOK_EXTENSIONS)
 
 class FileOrFolder(object):
 
     def __init__(self, entry, fs_cache):
+        self.all_storage_ids = fs_cache.all_storage_ids
+
         self.object_id = entry['id']
         self.is_folder = entry['is_folder']
         self.storage_id = entry['storage_id']
@@ -28,7 +35,12 @@ class FileOrFolder(object):
         self.name = force_unicode(n, 'utf-8')
         self.persistent_id = entry.get('persistent_id', self.object_id)
         self.size = entry.get('size', 0)
-        self.all_storage_ids = fs_cache.all_storage_ids
+        md = entry.get('modified', 0)
+        try:
+            self.last_modified = datetime.fromtimestamp(md, local_tz)
+        except:
+            self.last_modified = datetime.fromtimestamp(0, local_tz)
+        self.last_modified = as_utc(self.last_modified)
 
         if self.storage_id not in self.all_storage_ids:
             raise ValueError('Storage id %s not valid for %s, valid values: %s'%(self.storage_id,
@@ -50,6 +62,9 @@ class FileOrFolder(object):
         if self.storage_id == self.object_id:
             self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
 
+        self.is_ebook = (not self.is_folder and
+                self.name.rpartition('.')[-1].lower() in bexts)
+
     def __repr__(self):
         name = 'Folder' if self.is_folder else 'File'
         try:
@@ -147,6 +162,13 @@ class FileOrFolder(object):
             parent = c
         return parent
 
+    @property
+    def mtp_relpath(self):
+        return tuple(x.lower() for x in self.full_path[1:])
+
+    @property
+    def mtp_id_path(self):
+        return 'mtp:::' + json.dumps(self.object_id) + ':::' + '/'.join(self.full_path)
 
 class FilesystemCache(object):
 
@@ -192,4 +214,24 @@ class FilesystemCache(object):
             if e.storage_id == storage_id:
                 return e
 
+    def iterebooks(self, storage_id):
+        for x in self.id_map.itervalues():
+            if x.storage_id == storage_id and x.is_ebook:
+                yield x
+
+    def resolve_mtp_id_path(self, path):
+        if not path.startswith('mtp:::'):
+            raise ValueError('%s is not a valid MTP path'%path)
+        parts = path.split(':::')
+        if len(parts) < 3:
+            raise ValueError('%s is not a valid MTP path'%path)
+        try:
+            object_id = json.loads(parts[1])
+        except:
+            raise ValueError('%s is not a valid MTP path'%path)
+        try:
+            return self.id_map[object_id]
+        except KeyError:
+            raise ValueError('No object found with MTP path: %s'%path)
+
 

src/calibre/devices/mtp/test.py

@@ -128,7 +128,7 @@ class TestDeviceInteraction(unittest.TestCase):
 
         raw2 = io.BytesIO()
         pc = ProgressCallback()
-        self.dev.get_file(f, raw2, callback=pc)
+        self.dev.get_mtp_file(f, raw2, callback=pc)
         self.assertEqual(raw.getvalue(), raw2.getvalue())
         self.assertTrue(pc.end_called,
                 msg='Progress callback not called with equal values (get_file)')
@@ -162,7 +162,7 @@ class TestDeviceInteraction(unittest.TestCase):
         self.assertEqual(f.storage_id, self.storage.storage_id)
 
         raw2 = io.BytesIO()
-        self.dev.get_file(f, raw2)
+        self.dev.get_mtp_file(f, raw2)
         self.assertEqual(raw.getvalue(), raw2.getvalue())
 
     def measure_memory_usage(self, repetitions, func, *args, **kwargs):
@@ -226,7 +226,7 @@ class TestDeviceInteraction(unittest.TestCase):
         def get_file(f):
             raw = io.BytesIO()
             pc = ProgressCallback()
-            self.dev.get_file(f, raw, callback=pc)
+            self.dev.get_mtp_file(f, raw, callback=pc)
             raw.truncate(0)
             del raw
             del pc

src/calibre/devices/mtp/unix/driver.py

@@ -17,7 +17,6 @@ from calibre.constants import plugins
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
-from calibre.devices.mtp.filesystem_cache import FilesystemCache
 
 MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
         'bcd serial manufacturer product')
@@ -83,6 +82,8 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def debug_managed_device_detection(self, devices_on_system, output):
+        if self.currently_connected_dev is not None:
+            return True
         p = partial(prints, file=output)
         if self.libmtp is None:
             err = plugins['libmtp'][1]
@@ -175,6 +176,7 @@ class MTP_DEVICE(MTPDeviceBase):
     @property
     def filesystem_cache(self):
         if self._filesystem_cache is None:
+            from calibre.devices.mtp.filesystem_cache import FilesystemCache
             with self.lock:
                 storage, all_items, all_errs = [], [], []
                 for sid, capacity in zip([self._main_id, self._carda_id,
@@ -271,7 +273,7 @@ class MTP_DEVICE(MTPDeviceBase):
         return parent.add_child(ans)
 
     @synchronous
-    def get_file(self, f, stream=None, callback=None):
+    def get_mtp_file(self, f, stream=None, callback=None):
         if f.is_folder:
             raise ValueError('%s if a folder'%(f.full_path,))
         if stream is None:

src/calibre/devices/mtp/unix/libmtp.c

@@ -121,12 +121,13 @@ static uint16_t data_from_python(void *params, void *priv, uint32_t wantlen, uns
 static PyObject* build_file_metadata(LIBMTP_file_t *nf, uint32_t storage_id) {
     PyObject *ans = NULL;
 
-    ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:O}", 
+    ans = Py_BuildValue("{s:s, s:k, s:k, s:k, s:K, s:L, s:O}", 
             "name", (unsigned long)nf->filename,
             "id", (unsigned long)nf->item_id,
             "parent_id", (unsigned long)nf->parent_id,
             "storage_id", (unsigned long)storage_id,
             "size", nf->filesize,
+            "modified", (PY_LONG_LONG)nf->modificationdate,
             "is_folder", (nf->filetype == LIBMTP_FILETYPE_FOLDER) ? Py_True : Py_False
     );
 

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -34,6 +34,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
     ADDPROP(WPD_OBJECT_ISHIDDEN);
     ADDPROP(WPD_OBJECT_CAN_DELETE);
     ADDPROP(WPD_OBJECT_SIZE);
+    ADDPROP(WPD_OBJECT_DATE_MODIFIED);
 
     return properties;
 
@@ -81,6 +82,16 @@ static void set_size_property(PyObject *dict, REFPROPERTYKEY key, const char *py
     }
 }
 
+static void set_date_property(PyObject *dict, REFPROPERTYKEY key, const char *pykey, IPortableDeviceValues *properties) {
+    FLOAT val = 0;
+    PyObject *t;
+
+    if (SUCCEEDED(properties->GetFloatValue(key, &val))) {
+        t = Py_BuildValue("d", (double)val);
+        if (t != NULL) { PyDict_SetItemString(dict, pykey, t); Py_DECREF(t); }
+    }
+}
+
 static void set_content_type_property(PyObject *dict, IPortableDeviceValues *properties) {
     GUID guid = GUID_NULL;
     BOOL is_folder = 0;
@@ -103,6 +114,8 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
     set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values);
 
     set_size_property(obj, WPD_OBJECT_SIZE, "size", values);
+    set_date_property(obj, WPD_OBJECT_DATE_MODIFIED, "modified", values);
+
 }
 
 // }}}

src/calibre/devices/mtp/windows/driver.py

@@ -7,8 +7,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import time, threading
+import time, threading, traceback
-from functools import wraps
+from functools import wraps, partial
 from future_builtins import zip
 from itertools import chain
 
@@ -17,12 +17,12 @@ from calibre.constants import plugins, __appname__, numeric_version
 from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase
-from calibre.devices.mtp.filesystem_cache import FilesystemCache
 
 class ThreadingViolation(Exception):
 
     def __init__(self):
-        Exception.__init__('You cannot use the MTP driver from a thread other than the '
+        Exception.__init__(self,
+                'You cannot use the MTP driver from a thread other than the '
                 ' thread in which startup() was called')
 
 def same_thread(func):
@@ -51,6 +51,7 @@ class MTP_DEVICE(MTPDeviceBase):
         self._main_id = self._carda_id = self._cardb_id = None
         self.start_thread = None
         self._filesystem_cache = None
+        self.eject_dev_on_next_scan = False
 
     def startup(self):
         self.start_thread = threading.current_thread()
@@ -75,6 +76,10 @@ class MTP_DEVICE(MTPDeviceBase):
     @same_thread
     def detect_managed_devices(self, devices_on_system, force_refresh=False):
         if self.wpd is None: return None
+        if self.eject_dev_on_next_scan:
+            self.eject_dev_on_next_scan = False
+            if self.currently_connected_pnp_id is not None:
+                self.do_eject()
 
         devices_on_system = frozenset(devices_on_system)
         if (force_refresh or
@@ -124,6 +129,54 @@ class MTP_DEVICE(MTPDeviceBase):
 
         return None
 
+    @same_thread
+    def debug_managed_device_detection(self, devices_on_system, output):
+        import pprint
+        p = partial(prints, file=output)
+        if self.currently_connected_pnp_id is not None:
+            return True
+        if self.wpd_error:
+            p('Cannot detect MTP devices')
+            p(self.wpd_error)
+            return False
+        try:
+            pnp_ids = frozenset(self.wpd.enumerate_devices())
+        except:
+            p("Failed to get list of PNP ids on system")
+            p(traceback.format_exc())
+            return False
+
+        for pnp_id in pnp_ids:
+            try:
+                data = self.wpd.device_info(pnp_id)
+            except:
+                p('Failed to get data for device:', pnp_id)
+                p(traceback.format_exc())
+                continue
+            protocol = data.get('protocol', '').lower()
+            if not protocol.startswith('mtp:'): continue
+            p('MTP device:', pnp_id)
+            p(pprint.pformat(data))
+            if not self.is_suitable_wpd_device(data):
+                p('Not a suitable MTP device, ignoring\n')
+                continue
+            p('\nTrying to open:', pnp_id)
+            try:
+                self.open(pnp_id, 'debug-detection')
+            except:
+                p('Open failed:')
+                p(traceback.format_exc())
+                continue
+            break
+        if self.currently_connected_pnp_id:
+            p('Opened', self.current_friendly_name, 'successfully')
+            p('Device info:')
+            p(pprint.pformat(self.dev.data))
+            self.eject()
+            return True
+        p('No suitable MTP devices found')
+        return False
+
     def is_suitable_wpd_device(self, devdata):
         # Check that protocol is MTP
         protocol = devdata.get('protocol', '').lower()
@@ -143,6 +196,7 @@ class MTP_DEVICE(MTPDeviceBase):
     @property
     def filesystem_cache(self):
         if self._filesystem_cache is None:
+            from calibre.devices.mtp.filesystem_cache import FilesystemCache
             ts = self.total_space()
             all_storage = []
             items = []
@@ -164,19 +218,24 @@ class MTP_DEVICE(MTPDeviceBase):
         return self._filesystem_cache
 
     @same_thread
-    def post_yank_cleanup(self):
+    def do_eject(self):
-        self.currently_connected_pnp_id = self.current_friendly_name = None
-        self._main_id = self._carda_id = self._cardb_id = None
-        self.dev = self._filesystem_cache = None
-
-    @same_thread
-    def eject(self):
         if self.currently_connected_pnp_id is None: return
         self.ejected_devices.add(self.currently_connected_pnp_id)
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
         self.dev = self._filesystem_cache = None
 
+
+    @same_thread
+    def post_yank_cleanup(self):
+        self.currently_connected_pnp_id = self.current_friendly_name = None
+        self._main_id = self._carda_id = self._cardb_id = None
+        self.dev = self._filesystem_cache = None
+
+    def eject(self):
+        if self.currently_connected_pnp_id is None: return
+        self.eject_dev_on_next_scan = True
+
     @same_thread
     def open(self, connected_device, library_uuid):
         self.dev = self._filesystem_cache = None
@@ -200,7 +259,9 @@ class MTP_DEVICE(MTPDeviceBase):
             self._carda_id = storage[1]['id']
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
-        self.current_friendly_name = devdata.get('friendly_name', None)
+        self.current_friendly_name = devdata.get('friendly_name',
+                _('Unknown MTP device'))
+        self.currently_connected_pnp_id = connected_device
 
     @same_thread
     def get_basic_device_information(self):
@@ -232,7 +293,7 @@ class MTP_DEVICE(MTPDeviceBase):
         return tuple(ans)
 
     @same_thread
-    def get_file(self, f, stream=None, callback=None):
+    def get_mtp_file(self, f, stream=None, callback=None):
         if f.is_folder:
             raise ValueError('%s if a folder'%(f.full_path,))
         if stream is None:

src/calibre/devices/usbms/device.py

@@ -19,7 +19,7 @@ from calibre.devices.errors import (DeviceError, FreeSpaceError,
         WrongDestinationError)
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.constants import iswindows, islinux, isosx, isfreebsd, plugins
-from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
+from calibre.utils.filenames import ascii_filename as sanitize
 
 if isosx:
     usbobserver, usbobserver_err = plugins['usbobserver']
@@ -1052,78 +1052,16 @@ class Device(DeviceConfig, DevicePlugin):
         pass
 
     def create_upload_path(self, path, mdata, fname, create_dirs=True):
-        path = os.path.abspath(path)
+        from calibre.devices import create_upload_path
-        maxlen = self.MAX_PATH_LEN
-
-        special_tag = None
-        if mdata.tags:
-            for t in mdata.tags:
-                if t.startswith(_('News')) or t.startswith('/'):
-                    special_tag = t
-                    break
-
         settings = self.settings()
-        template = self.save_template()
+        filepath = create_upload_path(mdata, fname, self.save_template(), sanitize,
-        if mdata.tags and _('News') in mdata.tags:
+                prefix_path=os.path.abspath(path),
-            try:
+                maxlen=self.MAX_PATH_LEN,
-                p = mdata.pubdate
+                use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs,
-                date  = (p.year, p.month, p.day)
+                news_in_folder = self.NEWS_IN_FOLDER,
-            except:
+                filename_callback=self.filename_callback,
-                today = time.localtime()
+                sanitize_path_components=self.sanitize_path_components
-                date = (today[0], today[1], today[2])
+                )
-            template = "{title}_%d-%d-%d" % date
-        use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
-
-        fname = sanitize(fname)
-        ext = os.path.splitext(fname)[1]
-
-        from calibre.library.save_to_disk import get_components
-        from calibre.library.save_to_disk import config
-        opts = config().parse()
-        if not isinstance(template, unicode):
-            template = template.decode('utf-8')
-        app_id = str(getattr(mdata, 'application_id', ''))
-        id_ = mdata.get('id', fname)
-        extra_components = get_components(template, mdata, id_,
-                timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1)
-        if not extra_components:
-            extra_components.append(sanitize(self.filename_callback(fname,
-                mdata)))
-        else:
-            extra_components[-1] = sanitize(self.filename_callback(extra_components[-1]+ext, mdata))
-
-        if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
-            extra_components[-1] = 'x' + extra_components[-1][1:]
-
-        if special_tag is not None:
-            name = extra_components[-1]
-            extra_components = []
-            tag = special_tag
-            if tag.startswith(_('News')):
-                if self.NEWS_IN_FOLDER:
-                    extra_components.append('News')
-            else:
-                for c in tag.split('/'):
-                    c = sanitize(c)
-                    if not c: continue
-                    extra_components.append(c)
-            extra_components.append(name)
-
-        if not use_subdirs:
-            extra_components = extra_components[-1:]
-
-        def remove_trailing_periods(x):
-            ans = x
-            while ans.endswith('.'):
-                ans = ans[:-1].strip()
-            if not ans:
-                ans = 'x'
-            return ans
-
-        extra_components = list(map(remove_trailing_periods, extra_components))
-        components = shorten_components_to(maxlen - len(path), extra_components)
-        components = self.sanitize_path_components(components)
-        filepath = os.path.join(path, *components)
         filedir = os.path.dirname(filepath)
 
         if create_dirs and not os.path.exists(filedir):

src/calibre/devices/usbms/driver.py

@@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented
 for a particular device.
 '''
 
-import os, re, time, json, functools, shutil
+import os, time, json, shutil
 from itertools import cycle
 
 from calibre.constants import numeric_version
@@ -404,25 +404,8 @@ class USBMS(CLI, Device):
 
     @classmethod
     def build_template_regexp(cls):
-        def replfunc(match, seen=None):
+        from calibre.devices import build_template_regexp
-            v = match.group(1)
+        return build_template_regexp(cls.save_template())
-            if v in ['authors', 'author_sort']:
-                v = 'author'
-            if v in ('title', 'series', 'series_index', 'isbn', 'author'):
-                if v not in seen:
-                    seen.add(v)
-                    return '(?P<' + v + '>.+?)'
-            return '(.+?)'
-        s = set()
-        f = functools.partial(replfunc, seen=s)
-        template = None
-        try:
-            template = cls.save_template().rpartition('/')[2]
-            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
-        except:
-            prints(u'Failed to parse template: %r'%template)
-            template = u'{title} - {authors}'
-            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
 
     @classmethod
     def path_to_unicode(cls, path):

src/calibre/ebooks/metadata/opf2.py

@@ -510,6 +510,7 @@ class OPF(object): # {{{
     tags_path       = XPath('descendant::*[re:match(name(), "subject", "i")]')
     isbn_path       = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@scheme, "isbn", "i") or re:match(@opf:scheme, "isbn", "i"))]')
+    pubdate_path    = XPath('descendant::*[re:match(name(), "date", "i")]')
     raster_cover_path = XPath('descendant::*[re:match(name(), "meta", "i") and ' +
             're:match(@name, "cover", "i") and @content]')
     identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]')
@@ -538,8 +539,6 @@ class OPF(object): # {{{
                                         formatter=float, none_is=1)
     title_sort      = TitleSortField('title_sort', is_dc=False)
     rating          = MetadataField('rating', is_dc=False, formatter=float)
-    pubdate         = MetadataField('date', formatter=parse_date,
-                                    renderer=isoformat)
     publication_type = MetadataField('publication_type', is_dc=False)
     timestamp       = MetadataField('timestamp', is_dc=False,
                                     formatter=parse_date, renderer=isoformat)
@@ -852,6 +851,44 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
+    @dynamic_property
+    def pubdate(self):
+
+        def fget(self):
+            ans = None
+            for match in self.pubdate_path(self.metadata):
+                try:
+                    val = parse_date(etree.tostring(match, encoding=unicode,
+                        method='text', with_tail=False).strip())
+                except:
+                    continue
+                if ans is None or val < ans:
+                    ans = val
+            return ans
+
+        def fset(self, val):
+            least_val = least_elem = None
+            for match in self.pubdate_path(self.metadata):
+                try:
+                    cval = parse_date(etree.tostring(match, encoding=unicode,
+                        method='text', with_tail=False).strip())
+                except:
+                    match.getparent().remove(match)
+                else:
+                    if not val:
+                        match.getparent().remove(match)
+                    if least_val is None or cval < least_val:
+                        least_val, least_elem = cval, match
+
+            if val:
+                if least_val is None:
+                    least_elem = self.create_metadata_element('date')
+
+                least_elem.attrib.clear()
+                least_elem.text = isoformat(val)
+
+        return property(fget=fget, fset=fset)
+
     @dynamic_property
     def isbn(self):
 

src/calibre/ebooks/pdf/outline_writer.py

@@ -35,26 +35,31 @@ class Outline(object):
         page, ypos = 0, 0
         item = getattr(toc, 'outline_item_', None)
         if item is not None:
+            # First use the item URL without fragment
+            page, ypos = self.pos_map.get(item, {}).get(None, (0, 0))
             if toc.fragment:
                 amap = self.pos_map.get(item, None)
                 if amap is not None:
-                    page, ypos = amap.get(toc.fragment, (0, 0))
+                    page, ypos = amap.get(toc.fragment, (page, ypos))
-            else:
-                page, ypos = self.pos_map.get(item, {}).get(None, (0, 0))
         return page, ypos
 
     def add_children(self, toc, parent):
         for child in toc:
             page, ypos = self.get_pos(child)
             text = child.text or _('Page %d')%page
+            if page >= self.page_count:
+                page = self.page_count - 1
             cn = parent.create(text, page, True)
             self.add_children(child, cn)
 
     def __call__(self, doc):
         self.pos_map = dict(self.pos_map)
+        self.page_count = doc.page_count()
         for child in self.toc:
             page, ypos = self.get_pos(child)
             text = child.text or _('Page %d')%page
+            if page >= self.page_count:
+                page = self.page_count - 1
             node = doc.create_outline(text, page)
             self.add_children(child, node)
 

src/calibre/ebooks/pdf/writer.py

@@ -121,7 +121,7 @@ class PDFMetadata(object): # {{{
         self.author = force_unicode(self.author)
 # }}}
 
-class Page(QWebPage):
+class Page(QWebPage): # {{{
 
     def __init__(self, opts, log):
         self.log = log
@@ -137,10 +137,14 @@ class Page(QWebPage):
         std = {'serif':opts.pdf_serif_family, 'sans':opts.pdf_sans_family,
                 'mono':opts.pdf_mono_family}.get(opts.pdf_standard_font,
                         opts.pdf_serif_family)
+        if std:
             settings.setFontFamily(QWebSettings.StandardFont, std)
+        if opts.pdf_serif_family:
             settings.setFontFamily(QWebSettings.SerifFont, opts.pdf_serif_family)
+        if opts.pdf_sans_family:
             settings.setFontFamily(QWebSettings.SansSerifFont,
                     opts.pdf_sans_family)
+        if opts.pdf_mono_family:
             settings.setFontFamily(QWebSettings.FixedFont, opts.pdf_mono_family)
 
     def javaScriptConsoleMessage(self, msg, lineno, msgid):
@@ -148,6 +152,7 @@ class Page(QWebPage):
 
     def javaScriptAlert(self, frame, msg):
         self.log(unicode(msg))
+# }}}
 
 class PDFWriter(QObject): # {{{
 
@@ -192,6 +197,7 @@ class PDFWriter(QObject): # {{{
         self.insert_cover()
 
         self.render_succeeded = False
+        self.current_page_num = self.doc.page_count()
         self.combine_queue.append(os.path.join(self.tmp_path,
             'qprinter_out.pdf'))
         self.first_page = True
@@ -279,9 +285,13 @@ class PDFWriter(QObject): # {{{
         paged_display.fit_images();
         ''')
         mf = self.view.page().mainFrame()
+        start_page = self.current_page_num
+        if not self.first_page:
+            start_page += 1
         while True:
             if not self.first_page:
-                self.printer.newPage()
+                if self.printer.newPage():
+                    self.current_page_num += 1
             self.first_page = False
             mf.render(self.painter)
             nsl = evaljs('paged_display.next_screen_location()').toInt()
@@ -293,11 +303,10 @@ class PDFWriter(QObject): # {{{
         amap = self.bridge_value
         if not isinstance(amap, dict):
             amap = {} # Some javascript error occurred
-        pages = self.doc.page_count()
+        self.outline.set_pos(self.current_item, None, start_page, 0)
-        self.outline.set_pos(self.current_item, None, pages, 0)
         for anchor, x in amap.iteritems():
             pagenum, ypos = x
-            self.outline.set_pos(self.current_item, anchor, pages + pagenum, ypos)
+            self.outline.set_pos(self.current_item, anchor, start_page + pagenum, ypos)
 
     def append_doc(self, outpath):
         doc = self.podofo.PDFDoc()
@@ -342,8 +351,7 @@ class PDFWriter(QObject): # {{{
             if self.metadata.tags:
                 self.doc.keywords = self.metadata.tags
             self.outline(self.doc)
-            raw = self.doc.write()
+            self.doc.save_to_fileobj(self.out_stream)
-            self.out_stream.write(raw)
             self.render_succeeded = True
         finally:
             self._delete_tmpdir()

src/calibre/gui2/__init__.py

@@ -101,6 +101,7 @@ gprefs.defaults['auto_add_auto_convert'] = True
 gprefs.defaults['ui_style'] = 'calibre' if iswindows or isosx else 'system'
 gprefs.defaults['tag_browser_old_look'] = False
 gprefs.defaults['book_list_tooltips'] = True
+gprefs.defaults['bd_show_cover'] = True
 # }}}
 
 NONE = QVariant() #: Null value to return from the data function of item models

src/calibre/gui2/actions/add.py

@@ -10,9 +10,9 @@ from functools import partial
 
 from PyQt4.Qt import QPixmap, QTimer
 
-
+from calibre import as_unicode
-from calibre.gui2 import error_dialog, choose_files, \
+from calibre.gui2 import (error_dialog, choose_files, choose_dir,
-    choose_dir, warning_dialog, info_dialog
+        warning_dialog, info_dialog)
 from calibre.gui2.dialogs.add_empty_book import AddEmptyBookDialog
 from calibre.gui2.dialogs.progress import ProgressDialog
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
@@ -400,12 +400,45 @@ class AddAction(InterfaceAction):
             d = error_dialog(self.gui, _('Add to library'), _('No book files found'))
             d.exec_()
             return
-        paths = self.gui.device_manager.device.prepare_addable_books(paths)
+
+        self.gui.device_manager.prepare_addable_books(self.Dispatcher(partial(
+            self.books_prepared, view)), paths)
+        self.bpd = ProgressDialog(_('Downloading books'),
+                msg=_('Downloading books from device'), parent=self.gui,
+                cancelable=False)
+        QTimer.singleShot(1000, self.show_bpd)
+
+    def show_bpd(self):
+        if self.bpd is not None:
+            self.bpd.show()
+
+    def books_prepared(self, view, job):
+        self.bpd.hide()
+        self.bpd = None
+        if job.exception is not None:
+            self.gui.device_job_exception(job)
+            return
+        paths = job.result
+        ok_paths = [x for x in paths if isinstance(x, basestring)]
+        failed_paths = [x for x in paths if isinstance(x, tuple)]
+        if failed_paths:
+            if not ok_paths:
+                msg = _('Could not download files from the device')
+                typ = error_dialog
+            else:
+                msg = _('Could not download some files from the device')
+                typ = warning_dialog
+            det_msg = [x[0]+ '\n    ' + as_unicode(x[1]) for x in failed_paths]
+            det_msg = '\n\n'.join(det_msg)
+            typ(self.gui, _('Could not download files'), msg, det_msg=det_msg,
+                    show=True)
+
+        if ok_paths:
             from calibre.gui2.add import Adder
             self.__adder_func = partial(self._add_from_device_adder, on_card=None,
                                                         model=view.model())
             self._adder = Adder(self.gui, self.gui.library_view.model().db,
                     self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server)
-        self._adder.add(paths)
+            self._adder.add(ok_paths)
 
 

src/calibre/gui2/actions/view.py

@@ -256,6 +256,15 @@ class ViewAction(InterfaceAction):
             db.prefs['gui_view_history'] = history[:vh]
             self.build_menus(db)
 
+    def view_device_book(self, path):
+        pt = PersistentTemporaryFile('_view_device_book'+\
+                os.path.splitext(path)[1])
+        self.persistent_files.append(pt)
+        pt.close()
+        self.gui.device_manager.view_book(
+                Dispatcher(self.book_downloaded_for_viewing),
+                                        path, pt.name)
+
     def _view_books(self, rows):
         if not rows or len(rows) == 0:
             self._launch_viewer()
@@ -270,12 +279,5 @@ class ViewAction(InterfaceAction):
         else:
             paths = self.gui.current_view().model().paths(rows)
             for path in paths:
-                pt = PersistentTemporaryFile('_viewer_'+\
+                self.view_device_book(path)
-                        os.path.splitext(path)[1])
-                self.persistent_files.append(pt)
-                pt.close()
-                self.gui.device_manager.view_book(\
-                        Dispatcher(self.book_downloaded_for_viewing),
-                                              path, pt.name)
-
 

src/calibre/gui2/book_details.py

@@ -19,8 +19,8 @@ from calibre.ebooks.metadata import fmt_sidx
 from calibre.ebooks.metadata.sources.identify import urls_from_identifiers
 from calibre.constants import filesystem_encoding
 from calibre.library.comments import comments_to_html
-from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data,
+from calibre.gui2 import (config, open_url, pixmap_to_data, gprefs,
-        gprefs, rating_font)
+        rating_font)
 from calibre.utils.icu import sort_key
 from calibre.utils.formatter import EvalFormatter
 from calibre.utils.date import is_date_undefined
@@ -297,7 +297,8 @@ class CoverView(QWidget): # {{{
             self.pixmap = self.default_pixmap
         self.do_layout()
         self.update()
-        if not same_item and not config['disable_animations']:
+        if (not same_item and not config['disable_animations'] and
+                self.isVisible()):
             self.animation.start()
 
     def paintEvent(self, event):
@@ -512,6 +513,7 @@ class DetailsLayout(QLayout): # {{{
         self.do_layout(r)
 
     def cover_height(self, r):
+        if not self._children[0].widget().isVisible(): return 0
         mh = min(int(r.height()/2.), int(4/3. * r.width())+1)
         try:
             ph = self._children[0].widget().pixmap.height()
@@ -522,6 +524,7 @@ class DetailsLayout(QLayout): # {{{
         return mh
 
     def cover_width(self, r):
+        if not self._children[0].widget().isVisible(): return 0
         mw = 1 + int(3/4. * r.height())
         try:
             pw = self._children[0].widget().pixmap.width()
@@ -566,6 +569,7 @@ class BookDetails(QWidget): # {{{
     files_dropped = pyqtSignal(object, object)
     cover_changed = pyqtSignal(object, object)
     cover_removed = pyqtSignal(object)
+    view_device_book = pyqtSignal(object)
 
     # Drag 'n drop {{{
     DROPABBLE_EXTENSIONS = IMAGE_EXTENSIONS+BOOK_EXTENSIONS
@@ -640,7 +644,7 @@ class BookDetails(QWidget): # {{{
             id_, fmt = val.split(':')
             self.view_specific_format.emit(int(id_), fmt)
         elif typ == 'devpath':
-            open_local_file(val)
+            self.view_device_book.emit(val)
         else:
             try:
                 open_url(QUrl(link, QUrl.TolerantMode))
@@ -660,6 +664,7 @@ class BookDetails(QWidget): # {{{
         self.update_layout()
 
     def update_layout(self):
+        self.cover_view.setVisible(gprefs['bd_show_cover'])
         self._layout.do_layout(self.rect())
         self.cover_view.update_tooltip(self.current_path)
 

src/calibre/gui2/convert/__init__.py

@@ -143,7 +143,7 @@ class Widget(QWidget):
                 ans = None
             return ans
         elif isinstance(g, QFontComboBox):
-            ans = unicode(QFontInfo(g.currentFont().family()))
+            return unicode(QFontInfo(g.currentFont()).family())
         elif isinstance(g, EncodingComboBox):
             ans = unicode(g.currentText()).strip()
             try:

src/calibre/gui2/device.py

@@ -128,6 +128,10 @@ class DeviceManager(Thread): # {{{
         self.setDaemon(True)
         # [Device driver, Showing in GUI, Ejected]
         self.devices        = list(device_plugins())
+        self.managed_devices = [x for x in self.devices if
+                not x.MANAGES_DEVICE_PRESENCE]
+        self.unmanaged_devices = [x for x in self.devices if
+                x.MANAGES_DEVICE_PRESENCE]
         self.sleep_time     = sleep_time
         self.connected_slot = connected_slot
         self.jobs           = Queue.Queue(0)
@@ -182,11 +186,14 @@ class DeviceManager(Thread): # {{{
                     prints('Unable to open device', str(dev))
                     prints(tb)
                 continue
+            self.after_device_connect(dev, device_kind)
+            return True
+        return False
+
+    def after_device_connect(self, dev, device_kind):
         self.connected_device = dev
         self.connected_device_kind = device_kind
         self.connected_slot(True, device_kind)
-            return True
-        return False
 
     def connected_device_removed(self):
         while True:
@@ -215,7 +222,13 @@ class DeviceManager(Thread): # {{{
 
     def detect_device(self):
         self.scanner.scan()
+
         if self.is_device_connected:
+            if self.connected_device.MANAGES_DEVICE_PRESENCE:
+                cd = self.connected_device.detect_managed_devices(self.scanner.devices)
+                if cd is None:
+                    self.connected_device_removed()
+            else:
                 connected, detected_device = \
                     self.scanner.is_device_connected(self.connected_device,
                             only_presence=True)
@@ -228,9 +241,26 @@ class DeviceManager(Thread): # {{{
                                 only_presence=True, debug=True)
                     self.connected_device_removed()
         else:
+            for dev in self.unmanaged_devices:
+                try:
+                    cd = dev.detect_managed_devices(self.scanner.devices)
+                except:
+                    prints('Error during device detection for %s:'%dev)
+                    traceback.print_exc()
+                else:
+                    if cd is not None:
+                        try:
+                            dev.open(cd, self.current_library_uuid)
+                        except:
+                            prints('Error while trying to open %s (Driver: %s)'%
+                                    (cd, dev))
+                            traceback.print_exc()
+                        else:
+                            self.after_device_connect(dev, 'unmanaged-device')
+                            return
             try:
                 possibly_connected_devices = []
-                for device in self.devices:
+                for device in self.managed_devices:
                     if device in self.ejected_devices:
                         continue
                     try:
@@ -248,7 +278,7 @@ class DeviceManager(Thread): # {{{
                             prints('Connect to device failed, retrying in 5 seconds...')
                         time.sleep(5)
                         if not self.do_connect(possibly_connected_devices,
-                                           device_kind='usb'):
+                                           device_kind='device'):
                             if DEBUG:
                                 prints('Device connect failed again, giving up')
             except OpenFailed as e:
@@ -264,8 +294,9 @@ class DeviceManager(Thread): # {{{
     # disconnect a device
     def umount_device(self, *args):
         if self.is_device_connected and not self.job_manager.has_device_jobs():
-            if self.connected_device_kind == 'device':
+            if self.connected_device_kind in {'unmanaged-device', 'device'}:
                 self.connected_device.eject()
+                if self.connected_device_kind != 'unmanaged-device':
                     self.ejected_devices.add(self.connected_device)
                 self.connected_slot(False, self.connected_device_kind)
             elif hasattr(self.connected_device, 'unmount_device'):
@@ -412,6 +443,14 @@ class DeviceManager(Thread): # {{{
         return self.create_job_step(self._books, done,
                 description=_('Get list of books on device'), to_job=add_as_step_to_job)
 
+    def _prepare_addable_books(self, paths):
+        return self.device.prepare_addable_books(paths)
+
+    def prepare_addable_books(self, done, paths, add_as_step_to_job=None):
+        return self.create_job_step(self._prepare_addable_books, done, args=[paths],
+                description=_('Prepare files for transfer from device'),
+                to_job=add_as_step_to_job)
+
     def _annotations(self, path_map):
         return self.device.get_annotations(path_map)
 
@@ -525,9 +564,8 @@ class DeviceManager(Thread): # {{{
                         to_job=add_as_step_to_job)
 
     def _view_book(self, path, target):
-        f = open(target, 'wb')
+        with open(target, 'wb') as f:
             self.device.get_file(path, f)
-        f.close()
         return target
 
     def view_book(self, done, path, target, add_as_step_to_job=None):

src/calibre/gui2/init.py

@@ -269,6 +269,8 @@ class LayoutMixin(object): # {{{
                 self.iactions['Remove Books'].remove_format_by_id)
         self.book_details.save_specific_format.connect(
                 self.iactions['Save To Disk'].save_library_format_by_ids)
+        self.book_details.view_device_book.connect(
+                self.iactions['View'].view_device_book)
 
         m = self.library_view.model()
         if m.rowCount(None) > 0:

src/calibre/gui2/preferences/look_feel.py

@@ -106,6 +106,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                     'calibre')])
         r('book_list_tooltips', gprefs)
         r('tag_browser_old_look', gprefs, restart_required=True)
+        r('bd_show_cover', gprefs)
 
         r('cover_flow_queue_length', config, restart_required=True)
 

src/calibre/gui2/preferences/look_feel.ui

@@ -212,19 +212,32 @@
        <string>Book Details</string>
       </attribute>
       <layout class="QGridLayout" name="gridLayout_12">
-       <item row="1" column="0" rowspan="2">
+       <item row="2" column="1">
+        <widget class="QLabel" name="label_3">
+         <property name="text">
+          <string>Note that &lt;b&gt;comments&lt;/b&gt; will always be displayed at the end, regardless of the position you assign here.</string>
+         </property>
+         <property name="wordWrap">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <widget class="QCheckBox" name="opt_use_roman_numerals_for_series_number">
+         <property name="text">
+          <string>Use &amp;Roman numerals for series</string>
+         </property>
+         <property name="checked">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="0" rowspan="2">
         <widget class="QGroupBox" name="groupBox">
          <property name="title">
           <string>Select displayed metadata</string>
          </property>
          <layout class="QGridLayout" name="gridLayout_3">
-          <item row="0" column="0" rowspan="3">
-           <widget class="QListView" name="field_display_order">
-            <property name="alternatingRowColors">
-             <bool>true</bool>
-            </property>
-           </widget>
-          </item>
           <item row="0" column="1">
            <widget class="QToolButton" name="df_up_button">
             <property name="toolTip">
@@ -247,6 +260,13 @@
             </property>
            </widget>
           </item>
+          <item row="0" column="0" rowspan="3">
+           <widget class="QListView" name="field_display_order">
+            <property name="alternatingRowColors">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
           <item row="1" column="1">
            <spacer name="verticalSpacer_5">
             <property name="orientation">
@@ -288,23 +308,10 @@ Manage Authors. You can use the values {author} and
          </item>
         </layout>
        </item>
-       <item row="0" column="1">
+       <item row="1" column="0" colspan="2">
-        <widget class="QCheckBox" name="opt_use_roman_numerals_for_series_number">
+        <widget class="QCheckBox" name="opt_bd_show_cover">
          <property name="text">
-          <string>Use &amp;Roman numerals for series</string>
+          <string>Show &amp;cover in the book details panel</string>
-         </property>
-         <property name="checked">
-          <bool>true</bool>
-         </property>
-        </widget>
-       </item>
-       <item row="1" column="1">
-        <widget class="QLabel" name="label_3">
-         <property name="text">
-          <string>Note that &lt;b&gt;comments&lt;/b&gt; will always be displayed at the end, regardless of the position you assign here.</string>
-         </property>
-         <property name="wordWrap">
-          <bool>true</bool>
          </property>
         </widget>
        </item>

src/calibre/library/database2.py

@@ -32,7 +32,7 @@ from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename, samefile
 from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
-        parse_only_date)
+        parse_only_date, UNDEFINED_DATE)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
@@ -2498,7 +2498,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 self.notify('metadata', [id])
 
     def set_pubdate(self, id, dt, notify=True, commit=True):
-        if dt:
+        if not dt:
+            dt = UNDEFINED_DATE
         if isinstance(dt, basestring):
             dt = parse_only_date(dt)
         self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
@@ -3344,7 +3345,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.timestamp is None:
             mi.timestamp = utcnow()
         if mi.pubdate is None:
-            mi.pubdate = utcnow()
+            mi.pubdate = UNDEFINED_DATE
         self.set_metadata(id, mi, ignore_errors=True, commit=True)
         if cover is not None:
             try:
@@ -3386,7 +3387,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if mi.timestamp is None:
                 mi.timestamp = utcnow()
             if mi.pubdate is None:
-                mi.pubdate = utcnow()
+                mi.pubdate = UNDEFINED_DATE
             self.set_metadata(id, mi, commit=True, ignore_errors=True)
             npath = self.run_import_plugins(path, format)
             format = os.path.splitext(npath)[-1].lower().replace('.', '').upper()
@@ -3426,7 +3427,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.timestamp is None:
             mi.timestamp = utcnow()
         if mi.pubdate is None:
-            mi.pubdate = utcnow()
+            mi.pubdate = UNDEFINED_DATE
         self.set_metadata(id, mi, ignore_errors=True, commit=True)
         if preserve_uuid and mi.uuid:
             self.set_uuid(id, mi.uuid, commit=False)

src/calibre/utils/filenames.py

@@ -229,6 +229,10 @@ def samefile(src, dst):
     symlinks, case insensitivity, mapped drives, etc.
 
     Returns True iff both paths exist and point to the same file on disk.
+
+    Note: On windows will return True if the two string are identical (upto
+    case) even if the file does not exist. This is because I have no way of
+    knowing how reliable the GetFileInformationByHandle method is.
     '''
     if iswindows:
         return samefile_windows(src, dst)

src/calibre/utils/magick/magick.c

@@ -1104,6 +1104,41 @@ magick_Image_type_setter(magick_Image *self, PyObject *val, void *closure) {
 
 // }}}
 
+// Image.depth {{{
+static PyObject *
+magick_Image_depth_getter(magick_Image *self, void *closure) {
+   NULL_CHECK(NULL)
+
+    return Py_BuildValue("n", MagickGetImageDepth(self->wand));
+}
+
+static int
+magick_Image_depth_setter(magick_Image *self, PyObject *val, void *closure) {
+    size_t depth;
+
+    NULL_CHECK(-1)
+
+    if (val == NULL) {
+        PyErr_SetString(PyExc_TypeError, "Cannot delete image depth");
+        return -1;
+    }
+
+    if (!PyInt_Check(val)) {
+        PyErr_SetString(PyExc_TypeError, "Depth must be an integer");
+        return -1;
+    }
+
+    depth = (size_t)PyInt_AsSsize_t(val);
+    if (!MagickSetImageDepth(self->wand, depth)) {
+        PyErr_Format(PyExc_ValueError, "Could not set image depth to %lu", depth);
+        return -1;
+    }
+
+    return 0;
+}
+
+// }}}
+
 // Image.destroy {{{
 
 static PyObject *
@@ -1238,7 +1273,7 @@ static PyMethodDef magick_Image_methods[] = {
     },
 
     {"quantize", (PyCFunction)magick_Image_quantize, METH_VARARGS,
-     "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n nalyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." 
+     "quantize(number_colors, colorspace, treedepth, dither, measure_error) \n\n analyzes the colors within a reference image and chooses a fixed number of colors to represent the image. The goal of the algorithm is to minimize the color difference between the input and output image while minimizing the processing time." 
     },
 
     {NULL}  /* Sentinel */
@@ -1260,6 +1295,12 @@ static PyGetSetDef  magick_Image_getsetters[] = {
      (char *)"the image type: UndefinedType, BilevelType, GrayscaleType, GrayscaleMatteType, PaletteType, PaletteMatteType, TrueColorType, TrueColorMatteType, ColorSeparationType, ColorSeparationMatteType, or OptimizeType.",
      NULL},
 
+    {(char *)"depth",
+     (getter)magick_Image_depth_getter, (setter)magick_Image_depth_setter,
+     (char *)"the image depth.",
+     NULL},
+
+
     {NULL}  /* Sentinel */
 };
 

src/calibre/utils/podofo/__init__.py

@@ -94,9 +94,8 @@ def delete_all_but(path, pages):
         if page not in pages:
             p.delete_page(page)
 
-    raw = p.write()
     with open(path, 'wb') as f:
-        f.write(raw)
+        f.save_to_fileobj(path)
 
 def test_outline(src):
     podofo = get_podofo()
@@ -114,7 +113,17 @@ def test_outline(src):
         f.write(raw)
     print 'Outlined PDF:', out
 
+def test_save_to(src, dest):
+    podofo = get_podofo()
+    p = podofo.PDFDoc()
+    with open(src, 'rb') as f:
+        raw = f.read()
+    p.load(raw)
+    with open(dest, 'wb') as out:
+        p.save_to_fileobj(out)
+        print ('Wrote PDF of size:', out.tell())
+
 if __name__ == '__main__':
     import sys
-    test_outline(sys.argv[-1])
+    test_save_to(sys.argv[-2], sys.argv[-1])
 

src/calibre/utils/podofo/doc.cpp

@@ -104,6 +104,15 @@ PDFDoc_write(PDFDoc *self, PyObject *args) {
 
     return ans;
 }
+
+static PyObject *
+PDFDoc_save_to_fileobj(PDFDoc *self, PyObject *args) {
+    PyObject *f;
+
+    if (!PyArg_ParseTuple(args, "O", &f)) return NULL;
+    return write_doc(self->doc, f);
+}
+
 // }}}
 
 // extract_first_page() {{{
@@ -453,6 +462,9 @@ static PyMethodDef PDFDoc_methods[] = {
     {"write", (PyCFunction)PDFDoc_write, METH_VARARGS,
      "Return the PDF document as a bytestring."
     },
+    {"save_to_fileobj", (PyCFunction)PDFDoc_save_to_fileobj, METH_VARARGS,
+     "Write the PDF document to the soecified file-like object."
+    },
     {"extract_first_page", (PyCFunction)PDFDoc_extract_first_page, METH_VARARGS,
      "extract_first_page() -> Remove all but the first page."
     },

src/calibre/utils/podofo/global.h

@@ -41,6 +41,7 @@ extern void podofo_set_exception(const PdfError &err);
 extern PyObject * podofo_convert_pdfstring(const PdfString &s);
 extern PdfString * podofo_convert_pystring(PyObject *py);
 extern PdfString * podofo_convert_pystring_single_byte(PyObject *py);
+extern PyObject* write_doc(PdfMemDocument *doc, PyObject *f);
 
 }
 

src/calibre/utils/podofo/output.cpp

@@ -0,0 +1,177 @@
+/*
+ * output.cpp
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "global.h"
+
+using namespace PoDoFo;
+
+class pyerr : public std::exception {
+};
+
+class OutputDevice : public PdfOutputDevice {
+
+    private:
+        PyObject *file;
+        size_t written;
+
+        void update_written() {
+            size_t pos;
+            pos = Tell();
+            if (pos > written) written = pos;
+        }
+
+    public:
+        OutputDevice(PyObject *f) : file(f), written(0) { Py_XINCREF(file); }
+        ~OutputDevice() { Py_XDECREF(file); file = NULL; }
+
+        size_t GetLength() const { return written; }
+
+        long PrintVLen(const char* pszFormat, va_list args) {
+
+            if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); }
+
+#ifdef _MSC_VER
+            return _vscprintf(pszFormat, args);
+#else
+            char buf[10];
+            int res;
+            res = vsnprintf(buf, 1, pszFormat, args);
+            if (res < 0) {
+                PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf to get buffer length");
+                throw pyerr();
+            }
+            return static_cast<long>(res+1);
+#endif
+        }
+
+        void PrintV( const char* pszFormat, long lBytes, va_list args ) {
+            char *buf;
+            int res;
+
+            if( !pszFormat ) { PODOFO_RAISE_ERROR( ePdfError_InvalidHandle ); }
+
+            buf = new (std::nothrow) char[lBytes+1];
+            if (buf == NULL) { PyErr_NoMemory(); throw pyerr(); }
+            
+            // Note: PyOS_vsnprintf produces broken output on windows
+            res = vsnprintf(buf, lBytes, pszFormat, args);
+
+            if (res < 0) {
+                PyErr_SetString(PyExc_Exception, "Something bad happened while calling vsnprintf");
+                delete[] buf;
+                throw pyerr();
+            }
+
+            Write(buf, static_cast<size_t>(res));
+            delete[] buf;
+        }
+
+        void Print( const char* pszFormat, ... )
+        {
+            va_list args;
+            long lBytes;
+
+            va_start( args, pszFormat );
+            lBytes = PrintVLen(pszFormat, args);
+            va_end( args );
+
+            va_start( args, pszFormat );
+            PrintV(pszFormat, lBytes, args);
+            va_end( args );
+        }
+
+        size_t Read( char* pBuffer, size_t lLen ) {
+            PyObject *ret;
+            char *buf = NULL;
+            Py_ssize_t len = 0;
+
+            ret = PyObject_CallMethod(file, (char*)"read", (char*)"n", static_cast<Py_ssize_t>(lLen));
+            if (ret != NULL) {
+                if (PyBytes_AsStringAndSize(ret, &buf, &len) != -1) {
+                    memcpy(pBuffer, buf, len);
+                    Py_DECREF(ret);
+                    return static_cast<size_t>(len);
+                }
+                Py_DECREF(ret);
+            }
+
+            if (PyErr_Occurred() == NULL)
+                PyErr_SetString(PyExc_Exception, "Failed to read data from python file object");
+
+            throw pyerr();
+
+        }
+
+        void Seek(size_t offset) {
+            PyObject *ret;
+            ret = PyObject_CallMethod(file, (char*)"seek", (char*)"n", static_cast<Py_ssize_t>(offset));
+            if (ret == NULL) {
+                if (PyErr_Occurred() == NULL)
+                    PyErr_SetString(PyExc_Exception, "Failed to seek in python file object");
+                throw pyerr();
+            }
+            Py_DECREF(ret);
+        }
+
+        size_t Tell() const {
+            PyObject *ret;
+            unsigned long ans;
+
+            ret = PyObject_CallMethod(file, (char*)"tell", NULL);
+            if (ret == NULL) {
+                if (PyErr_Occurred() == NULL)
+                    PyErr_SetString(PyExc_Exception, "Failed to call tell() on python file object");
+                throw pyerr();
+            }
+            if (!PyNumber_Check(ret)) {
+                Py_DECREF(ret);
+                PyErr_SetString(PyExc_Exception, "tell() method did not return a number");
+                throw pyerr();
+            }
+            ans = PyInt_AsUnsignedLongMask(ret);
+            Py_DECREF(ret);
+            if (PyErr_Occurred() != NULL) throw pyerr();
+
+            return static_cast<size_t>(ans);
+        }
+
+        void Write(const char* pBuffer, size_t lLen) {
+            PyObject *ret;
+
+            ret = PyObject_CallMethod(file, (char*)"write", (char*)"s#", pBuffer, (int)lLen);
+            if (ret == NULL) {
+                if (PyErr_Occurred() == NULL)
+                    PyErr_SetString(PyExc_Exception, "Failed to call write() on python file object");
+                throw pyerr();
+            }
+            Py_DECREF(ret);
+            update_written();
+        }
+
+        void Flush() {
+            Py_XDECREF(PyObject_CallMethod(file, (char*)"flush", NULL));
+        }
+
+};
+
+
+PyObject* pdf::write_doc(PdfMemDocument *doc, PyObject *f) {
+    OutputDevice d(f);
+
+    try {
+        doc->Write(&d);
+    } catch(const PdfError & err) {
+        podofo_set_exception(err); return NULL;
+    } catch (...) {
+        if (PyErr_Occurred() == NULL) 
+            PyErr_SetString(PyExc_Exception, "An unknown error occurred while trying to write the pdf to the file object");
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+