Merge from trunk

recipes/le_monde.recipe

@@ -9,7 +9,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 class LeMonde(BasicNewsRecipe):
     title                  = 'Le Monde'
     __author__             = 'veezh'
-    description            = 'Actualités'
+    description            = u'Actualités'
     oldest_article         = 1
     max_articles_per_feed  = 100
     no_stylesheets         = True

recipes/le_monde_sub.recipe

@@ -0,0 +1,132 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Rémi Vanicat <vanicat at debian.org>'
+'''
+Lemonde.fr: Version abonnée
+'''
+
+
+import os, zipfile, re, time
+
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+from calibre.ptempfile import PersistentTemporaryFile
+
+class LeMondeAbonne(BasicNewsRecipe):
+
+    title                 = u'Le Monde: Édition abonnés'
+    __author__            = u'Rémi Vanicat'
+    description           = u'Actualités'
+    category              = u'Actualités, France, Monde'
+    language              = 'fr'
+    needs_subscription    = True
+
+    no_stylesheets         = True
+
+    extra_css = u'''
+                    h1{font-size:130%;}
+                    .ariane{font-size:xx-small;}
+                    .source{font-size:xx-small;}
+                    .href{font-size:xx-small;}
+                    .LM_caption{color:#666666; font-size:x-small;}
+                    .main-article-info{font-family:Arial,Helvetica,sans-serif;}
+                    #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                    #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                '''
+
+    zipurl_format = 'http://medias.lemonde.fr/abonnes/editionelectronique/%Y%m%d/html/%y%m%d.zip'
+    coverurl_format = '/img/%y%m%d01.jpg'
+    path_format = "%y%m%d"
+    login_url = 'http://www.lemonde.fr/web/journal_electronique/identification/1,56-0,45-0,0.html'
+
+    keep_only_tags = [ dict(name="div", attrs={ 'class': 'po-prti' }),  dict(name=['h1']), dict(name='div', attrs={ 'class': 'photo' }), dict(name='div', attrs={ 'class': 'po-ti2' }), dict(name='div', attrs={ 'class': 'ar-txt' }), dict(name='div', attrs={ 'class': 'po_rtcol' }) ]
+
+    article_id_pattern = re.compile("[0-9]+\\.html")
+    article_url_format = 'http://www.lemonde.fr/journalelectronique/donnees/protege/%Y%m%d/html/'
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.login_url)
+            br.select_form(nr=0)
+            br['login']    = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    decalage = 24 * 60 * 60     # today Monde has tomorow date
+
+    def get_cover_url(self):
+        url = time.strftime(self.coverurl_format, self.ltime)
+        return self.articles_path + url
+
+    def parse_index(self):
+        browser = self.get_browser()
+
+        second = time.time()
+        second += self.decalage
+        ltime = self.ltime = time.gmtime(second)
+        url = time.strftime(self.zipurl_format, ltime)
+
+        self.timefmt=strftime(" %A %d %B %Y", ltime)
+
+        response = browser.open(url)
+
+        tmp = PersistentTemporaryFile(suffix='.zip')
+        self.report_progress(0.1,_('downloading zip file'))
+        tmp.write(response.read())
+        tmp.close()
+
+        zfile = zipfile.ZipFile(tmp.name, 'r')
+        self.report_progress(0.1,_('extracting zip file'))
+
+        zfile.extractall(self.output_dir)
+        zfile.close()
+
+        path = os.path.join(self.output_dir, time.strftime(self.path_format, ltime), "data")
+
+        self.articles_path = path
+
+        files = os.listdir(path)
+
+        nb_index_files = len([ name for name in files if re.match("frame_gauche_[0-9]+.html", name) ])
+
+        flux = []
+
+        article_url = time.strftime(self.article_url_format, ltime)
+
+        for i in range(nb_index_files):
+            filename = os.path.join(path, "selection_%d.html" % (i + 1))
+            tmp = open(filename,'r')
+            soup=BeautifulSoup(tmp)
+            title=soup.find('span').contents[0]
+            tmp.close()
+
+            filename = os.path.join(path, "frame_gauche_%d.html" % (i + 1))
+            tmp = open(filename,'r')
+            soup = BeautifulSoup(tmp)
+            articles = []
+            for link in soup.findAll("a"):
+                article_file = link['href']
+                article_id=self.article_id_pattern.search(article_file).group()
+                article = {
+                    'title': link.contents[0],
+                    'url': article_url + article_id,
+                    'descripion': '',
+                    'content': ''
+                    }
+                articles.append(article)
+            tmp.close()
+
+            flux.append((title, articles))
+
+        return flux
+
+
+
+# Local Variables:
+# mode: python
+# End:
+

recipes/sunday_times.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010-2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.thesundaytimes.co.uk
 '''
@@ -43,13 +43,14 @@ class TimesOnline(BasicNewsRecipe):
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
-        br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/')
+        br.open('http://www.thesundaytimes.co.uk/sto/')
         if self.username is not None and self.password is not None:
-            data = urllib.urlencode({ 'userName':self.username
+            data = urllib.urlencode({ 
+                                      'gotoUrl' :self.INDEX
+                                     ,'username':self.username
                                      ,'password':self.password
-                                     ,'keepMeLoggedIn':'false'
                                    })
-            br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
+            br.open('https://acs.thetimes.co.uk/user/login',data)
         return br
 
     remove_tags      = [

recipes/times_online.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2009-2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2009-2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.thetimes.co.uk
 '''
@@ -21,6 +21,7 @@ class TimesOnline(BasicNewsRecipe):
     encoding              = 'utf-8'
     delay                 = 1
     needs_subscription    = True
+    auto_cleanup          = False    
     publication_type      = 'newspaper'
     masthead_url          = 'http://www.thetimes.co.uk/tto/public/img/the_times_460.gif'
     INDEX                 = 'http://www.thetimes.co.uk'
@@ -41,13 +42,14 @@ class TimesOnline(BasicNewsRecipe):
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
-        br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thetimes.co.uk/tto/news/?lightbox=false')
+        br.open('http://www.thetimes.co.uk/tto/news/')
         if self.username is not None and self.password is not None:
-            data = urllib.urlencode({ 'userName':self.username
+            data = urllib.urlencode({ 
+                                      'gotoUrl' :self.INDEX
+                                     ,'username':self.username
                                      ,'password':self.password
-                                     ,'keepMeLoggedIn':'false'
                                    })
-            br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
+            br.open('https://acs.thetimes.co.uk/user/login',data)
         return br
 
     remove_tags      = [
@@ -58,6 +60,7 @@ class TimesOnline(BasicNewsRecipe):
     keep_only_tags   = [
                           dict(attrs={'class':'heading' })
                          ,dict(attrs={'class':'f-author'})
+                         ,dict(attrs={'class':['media','byline-timestamp']})
                          ,dict(attrs={'id':'bodycopy'})
                        ]
 
@@ -79,11 +82,6 @@ class TimesOnline(BasicNewsRecipe):
                ,(u'Arts'        , PREFIX + u'arts/?view=list'         )
             ]
 
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        return self.adeify_images(soup)
-
     def parse_index(self):
         totalfeeds = []
         lfeeds = self.get_feeds()

recipes/variety.recipe

@@ -19,7 +19,13 @@ class Variety(BasicNewsRecipe):
     category               = 'Entertainment Industry News, Daily Variety, Movie Reviews, TV, Awards, Oscars, Cannes, Box Office, Hollywood'
     language               = 'en'
     masthead_url           = 'http://images1.variety.com/graphics/variety/Variety_logo_green_tm.gif'
-    extra_css              = ' body{font-family: Georgia,"Times New Roman",Times,Courier,serif } img{margin-bottom: 1em} '
+    extra_css              = """ 
+                                body{font-family: Arial,Helvetica,sans-serif; font-size: 1.275em}
+                                .date{font-size: small; border: 1px dotted rgb(204, 204, 204); font-style: italic; color: rgb(102, 102, 102); margin: 5px 0px; padding: 0.5em;}
+                                .author{margin: 5px 0px 5px 20px; padding: 0.5em; background: none repeat scroll 0% 0% rgb(247, 247, 247);}
+                                .art h2{color: rgb(153, 0, 0); font-size: 1.275em; font-weight: bold;}
+                                img{margin-bottom: 1em} 
+                             """
 
     conversion_options = {
                              'comments'  : description
@@ -29,7 +35,7 @@ class Variety(BasicNewsRecipe):
                          }
 
     remove_tags = [dict(name=['object','link','map'])]
-
+    remove_attributes=['lang','vspace','hspace','xmlns:ms','xmlns:dt']
     keep_only_tags = [dict(name='div', attrs={'class':'art control'})]
 
     feeds = [(u'News & Articles', u'http://feeds.feedburner.com/variety/headlines' )]
@@ -37,3 +43,29 @@ class Variety(BasicNewsRecipe):
     def print_version(self, url):
         rpt = url.rpartition('.html')[0]
         return rpt + '?printerfriendly=true'
+
+    def preprocess_raw_html(self, raw, url):
+       return '<html><head>'+raw[raw.find('</head>'):]
+
+    def get_article_url(self, article):
+        url = BasicNewsRecipe.get_article_url(self, article)
+        return url.rpartition('?')[0]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup
+        

resources/templates/html.css

@@ -402,7 +402,3 @@ img, object, svg|svg {
     height: auto;
 }
 
-/* These are needed because ADE renders anchors the same as links */
-
-a { text-decoration: inherit; color: inherit; cursor: inherit }
-a[href] { text-decoration: underline; color: blue; cursor: pointer }

src/calibre/__init__.py

@@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None):
 
     return filename
 
-def human_readable(size):
+def human_readable(size, sep=' '):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"
     for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
@@ -686,7 +686,7 @@ def human_readable(size):
         size = size[:size.find(".")+2]
     if size.endswith('.0'):
         size = size[:-2]
-    return size + " " + suffix
+    return size + sep + suffix
 
 def remove_bracketed_text(src,
         brackets={u'(':u')', u'[':u']', u'{':u'}'}):

src/calibre/devices/android/driver.py

@@ -197,7 +197,7 @@ class ANDROID(USBMS):
             'GENERIC-', 'ZTE', 'MID', 'QUALCOMM', 'PANDIGIT', 'HYSTON',
             'VIZIO', 'GOOGLE', 'FREESCAL', 'KOBO_INC', 'LENOVO', 'ROCKCHIP',
             'POCKET', 'ONDA_MID', 'ZENITHIN', 'INGENIC', 'PMID701C', 'PD',
-            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ']
+            'PMP5097C', 'MASS', 'NOVO7', 'ZEKI', 'COBY', 'SXZ', 'USB_2.0']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -212,7 +212,7 @@ class ANDROID(USBMS):
             'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107',
             'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855',
             'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW',
-            'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0_DRIVER',
+            'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD', 'USB_2.0',
             'GT-S5830L_CARD', 'UNIVERSE', 'XT875', 'PRO', '.KOBO_VOX',
             'THINKPAD_TABLET', 'SGH-T989', 'YP-G70', 'STORAGE_DEVICE',
             'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID',
@@ -224,7 +224,7 @@ class ANDROID(USBMS):
             'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853',
             'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD',
             'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
-            'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0_DRIVER', 'XT875',
+            'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875',
             'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727',
             'USB_FLASH_DRIVER', 'ANDROID', 'MID7042']
 

src/calibre/devices/mtp/filesystem_cache.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import weakref, sys
+from operator import attrgetter
+from future_builtins import map
+
+from calibre import human_readable, prints, force_unicode
+from calibre.utils.icu import sort_key
+
+class FileOrFolder(object):
+
+    def __init__(self, entry, fs_cache, all_storage_ids):
+        self.object_id = entry['id']
+        self.is_folder = entry['is_folder']
+        self.name = force_unicode(entry.get('name', '___'), 'utf-8')
+        self.persistent_id = entry.get('persistent_id', self.object_id)
+        self.size = entry.get('size', 0)
+        # self.parent_id is None for storage objects
+        self.parent_id = entry.get('parent_id', None)
+        if self.parent_id == 0:
+            sid = entry['storage_id']
+            if sid not in all_storage_ids:
+                sid = all_storage_ids[0]
+            self.parent_id = sid
+        self.is_hidden = entry.get('is_hidden', False)
+        self.is_system = entry.get('is_system', False)
+        self.can_delete = entry.get('can_delete', True)
+
+        self.files = []
+        self.folders = []
+        fs_cache.id_map[self.object_id] = self
+        self.fs_cache = weakref.ref(fs_cache)
+
+    @property
+    def id_map(self):
+        return self.fs_cache().id_map
+
+    @property
+    def parent(self):
+        return None if self.parent_id is None else self.id_map[self.parent_id]
+
+    def __iter__(self):
+        for e in self.folders:
+            yield e
+        for e in self.files:
+            yield e
+
+    def dump(self, prefix='', out=sys.stdout):
+        c = '+' if self.is_folder else '-'
+        data = ('%s children'%(sum(map(len, (self.files, self.folders))))
+            if self.is_folder else human_readable(self.size))
+        line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data)
+        prints(line, file=out)
+        for c in (self.folders, self.files):
+            for e in sorted(c, key=lambda x:sort_key(x.name)):
+                e.dump(prefix=prefix+'  ', out=out)
+
+class FilesystemCache(object):
+
+    def __init__(self, all_storage, entries):
+        self.entries = []
+        self.id_map = {}
+
+        for storage in all_storage:
+            e = FileOrFolder(storage, self, [])
+            self.entries.append(e)
+
+        self.entries.sort(key=attrgetter('object_id'))
+        all_storage_ids = [x.object_id for x in self.entries]
+
+        for entry in entries:
+            FileOrFolder(entry, self, all_storage_ids)
+
+        for item in self.id_map.itervalues():
+            p = item.parent
+            if p is not None:
+                t = p.folders if item.is_folder else p.files
+                t.append(item)
+
+    def dump(self, out=sys.stdout):
+        for e in self.entries:
+            e.dump(out=out)
+
+

src/calibre/devices/mtp/unix/driver.py

@@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en'
 
 import time, operator
 from threading import RLock
-from itertools import chain
-from collections import deque, OrderedDict
 from io import BytesIO
 
-from calibre import prints
 from calibre.devices.errors import OpenFailed, DeviceError
 from calibre.devices.mtp.base import MTPDeviceBase, synchronous
+from calibre.devices.mtp.filesystem_cache import FilesystemCache
 from calibre.devices.mtp.unix.detect import MTPDetect
 
-class FilesystemCache(object):
-
-    def __init__(self, files, folders):
-        self.files = files
-        self.folders = folders
-        self.file_id_map = {f['id']:f for f in files}
-        self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)}
-
-        # Set the parents of each file
-        self.files_in_root = OrderedDict()
-        for f in files:
-            parents = deque()
-            pid = f['parent_id']
-            while pid is not None and pid > 0:
-                try:
-                    parent = self.folder_id_map[pid]
-                except KeyError:
-                    break
-                parents.appendleft(pid)
-                pid = parent['parent_id']
-            f['parents'] = parents
-            if not parents:
-                self.files_in_root[f['id']] = f
-
-        # Set the files in each folder
-        for f in self.iterfolders():
-            f['files'] = [i for i in files if i['parent_id'] ==
-                    f['id']]
-
-        # Decode the file and folder names
-        for f in chain(files, folders):
-            try:
-                name = f['name'].decode('utf-8')
-            except UnicodeDecodeError:
-                name = 'undecodable_%d'%f['id']
-            f['name'] = name
-
-    def iterfolders(self, folders=None, set_level=None):
-        clevel = None if set_level is None else set_level + 1
-        if folders is None:
-            folders = self.folders
-        for f in folders:
-            if set_level is not None:
-                f['level'] = set_level
-            yield f
-            for c in f['children']:
-                for child in self.iterfolders([c], set_level=clevel):
-                    yield child
-
-    def dump_filesystem(self):
-        indent = 2
-        for f in self.iterfolders():
-            prefix = ' '*(indent*f['level'])
-            prints(prefix, '+', f['name'], 'id=%s'%f['id'])
-            for leaf in f['files']:
-                prints(prefix, ' '*indent, '-', leaf['name'],
-                        'id=%d'%leaf['id'], 'size=%d'%leaf['size'],
-                        'modtime=%d'%leaf['modtime'])
-        for leaf in self.files_in_root.itervalues():
-            prints('-', leaf['name'], 'id=%d'%leaf['id'],
-                    'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime'])
-
 class MTP_DEVICE(MTPDeviceBase):
 
     supported_platforms = ['linux']
@@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase):
     def __init__(self, *args, **kwargs):
         MTPDeviceBase.__init__(self, *args, **kwargs)
         self.dev = None
-        self.filesystem_cache = None
+        self._filesystem_cache = None
         self.lock = RLock()
         self.blacklisted_devices = set()
 
@@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def post_yank_cleanup(self):
-        self.dev = self.filesystem_cache = self.current_friendly_name = None
+        self.dev = self._filesystem_cache = self.current_friendly_name = None
 
     @synchronous
     def startup(self):
@@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def shutdown(self):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
     def format_errorstack(self, errs):
         return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for
@@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase):
 
     @synchronous
     def open(self, connected_device, library_uuid):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
         def blacklist_device():
             d = connected_device
             self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
@@ -179,23 +115,41 @@ class MTP_DEVICE(MTPDeviceBase):
             self._carda_id = storage[1]['id']
         if len(storage) > 2:
             self._cardb_id = storage[2]['id']
-        self.current_friendly_name = self.dev.name
+        self.current_friendly_name = self.dev.friendly_name
 
-    @synchronous
+    @property
-    def read_filesystem_cache(self):
+    def filesystem_cache(self):
-        try:
+        if self._filesystem_cache is None:
-            files, errs = self.dev.get_filelist(self)
+            with self.lock:
-            if errs and not files:
+                files, errs = self.dev.get_filelist(self)
-                raise DeviceError('Failed to read files from device. Underlying errors:\n'
+                if errs and not files:
-                        +self.format_errorstack(errs))
+                    raise DeviceError('Failed to read files from device. Underlying errors:\n'
-            folders, errs = self.dev.get_folderlist()
+                            +self.format_errorstack(errs))
-            if errs and not folders:
+                folders, errs = self.dev.get_folderlist()
-                raise DeviceError('Failed to read folders from device. Underlying errors:\n'
+                if errs and not folders:
-                        +self.format_errorstack(errs))
+                    raise DeviceError('Failed to read folders from device. Underlying errors:\n'
-            self.filesystem_cache = FilesystemCache(files, folders)
+                            +self.format_errorstack(errs))
-        except:
+                storage = []
-            self.dev = self._main_id = self._carda_id = self._cardb_id = None
+                for sid, capacity in zip([self._main_id, self._carda_id,
-            raise
+                    self._cardb_id], self.total_space()):
+                    if sid is not None:
+                        name = _('Unknown')
+                        for x in self.dev.storage_info:
+                            if x['id'] == sid:
+                                name = x['name']
+                                break
+                        storage.append({'id':sid, 'size':capacity,
+                            'is_folder':True, 'name':name})
+                all_folders = []
+                def recurse(f):
+                    all_folders.append(f)
+                    for c in f['children']:
+                        recurse(c)
+
+                for f in folders: recurse(f)
+                self._filesystem_cache = FilesystemCache(storage,
+                        all_folders+files)
+        return self._filesystem_cache
 
     @synchronous
     def get_device_information(self, end_session=True):
@@ -246,7 +200,6 @@ if __name__ == '__main__':
     devs = linux_scanner()
     mtp_devs = dev.detect(devs)
     dev.open(list(mtp_devs)[0], 'xxx')
-    dev.read_filesystem_cache()
     d = dev.dev
     print ("Opened device:", dev.get_gui_name())
     print ("Storage info:")
@@ -257,7 +210,7 @@ if __name__ == '__main__':
     # fname = b'moose.txt'
     # src = BytesIO(raw)
     # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR()))
-    dev.filesystem_cache.dump_filesystem()
+    dev.filesystem_cache.dump()
     # with open('/tmp/flint.epub', 'wb') as f:
     #     print(d.get_file(786, f, PR()))
     # print()

src/calibre/devices/mtp/unix/libmtp.c

@@ -55,7 +55,7 @@ static int report_progress(uint64_t const sent, uint64_t const total, void const
     cb = (ProgressCallback *)data;
     if (cb->obj != NULL) {
         PyEval_RestoreThread(cb->state);
-        res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
+        res = PyObject_CallFunction(cb->obj, "KK", sent, total);
         Py_XDECREF(res);
         cb->state = PyEval_SaveThread();
     }
@@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
                 "capacity", storage->MaxCapacity,
                 "freespace_bytes", storage->FreeSpaceInBytes,
                 "freespace_objects", storage->FreeSpaceInObjects,
-                "storage_desc", storage->StorageDescription,
+                "name", storage->StorageDescription,
                 "volume_id", storage->VolumeIdentifier
         );
 
@@ -339,6 +339,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
 
 
     if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
     cb.obj = callback;
 
     ans = PyList_New(0);
@@ -357,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
     }
 
     for (f=tf; f != NULL; f=f->next) {
-        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}",
                 "id", f->item_id,
                 "parent_id", f->parent_id,
                 "storage_id", f->storage_id,
                 "name", f->filename,
                 "size", f->filesize,
-                "modtime", f->modificationdate
+                "modtime", f->modificationdate,
+                "is_folder", Py_False
         );
         if (fo == NULL || PyList_Append(ans, fo) != 0) break;
         Py_DECREF(fo);
@@ -377,7 +379,7 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs
 
     if (callback != NULL) {
         // Bug in libmtp where it does not call callback with 100%
-        fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
+        fo = PyObject_CallFunction(callback, "KK", PyList_Size(ans), PyList_Size(ans));
         Py_XDECREF(fo);
     }
 
@@ -392,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
     children = PyList_New(0);
     if (children == NULL) { PyErr_NoMemory(); return 1;}
 
-    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}",
             "id", f->folder_id,
             "parent_id", f->parent_id,
             "storage_id", f->storage_id,
             "name", f->name,
+            "is_folder", Py_True,
             "children", children);
     if (folder == NULL) return 1;
     PyList_Append(parent, folder);
@@ -454,6 +457,7 @@ libmtp_Device_get_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
     if (!PyArg_ParseTuple(args, "kO|O", &fileid, &stream, &callback)) return NULL; 
     errs = PyList_New(0);
     if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
 
     cb.obj = callback; cb.extra = stream;
     Py_XINCREF(callback); Py_INCREF(stream);
@@ -486,6 +490,7 @@ libmtp_Device_put_file(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
     if (!PyArg_ParseTuple(args, "kksOK|O", &storage_id, &parent_id, &name, &stream, &filesize, &callback)) return NULL; 
     errs = PyList_New(0);
     if (errs == NULL) { PyErr_NoMemory(); return NULL; }
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
 
     cb.obj = callback; cb.extra = stream;
     f.parent_id = parent_id; f.storage_id = storage_id; f.item_id = 0; f.filename = name; f.filetype = LIBMTP_FILETYPE_UNKNOWN; f.filesize = filesize;
@@ -599,7 +604,7 @@ static PyMethodDef libmtp_Device_methods[] = {
     },
 
     {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
-     "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
+     "get_filelist(callback=None) -> Get the list of files on the device. callback must be callable accepts arguments (current, total)'. Returns files, errors."
     },
 
     {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,

src/calibre/devices/mtp/windows/content_enumeration.cpp

@@ -14,7 +14,7 @@
 namespace wpd {
 
 static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { // {{{
-    IPortableDeviceKeyCollection *properties;
+    IPortableDeviceKeyCollection *properties = NULL;
     HRESULT hr;
 
     Py_BEGIN_ALLOW_THREADS;
@@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() {
     ADDPROP(WPD_OBJECT_PARENT_ID);
     ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID);
     ADDPROP(WPD_OBJECT_NAME);
-    ADDPROP(WPD_OBJECT_SYNC_ID);
+    // ADDPROP(WPD_OBJECT_SYNC_ID);
     ADDPROP(WPD_OBJECT_ISSYSTEM);
     ADDPROP(WPD_OBJECT_ISHIDDEN);
     ADDPROP(WPD_OBJECT_CAN_DELETE);
@@ -87,8 +87,25 @@ static void set_content_type_property(PyObject *dict, IPortableDeviceValues *pro
     if (SUCCEEDED(properties->GetGuidValue(WPD_OBJECT_CONTENT_TYPE, &guid)) && IsEqualGUID(guid, WPD_CONTENT_TYPE_FOLDER)) is_folder = 1;
     PyDict_SetItemString(dict, "is_folder", (is_folder) ? Py_True : Py_False);
 }
+
+static void set_properties(PyObject *obj, IPortableDeviceValues *values) {
+    set_content_type_property(obj, values);
+
+    set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values);
+    set_string_property(obj, WPD_OBJECT_NAME, "name", values);
+    // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values);
+    set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values);
+
+    set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values);
+    set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", values);
+    set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", values);
+
+    set_size_property(obj, WPD_OBJECT_SIZE, "size", values);
+}
+
 // }}}
 
+// Bulk get filesystem {{{
 class GetBulkCallback : public IPortableDevicePropertiesBulkCallback {
 
 public:
@@ -154,19 +171,8 @@ public:
                     } 
                     Py_DECREF(temp);
 
-                    set_content_type_property(obj, properties);
+                    set_properties(obj, properties);
 
-                    set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", properties);
-                    set_string_property(obj, WPD_OBJECT_NAME, "name", properties);
-                    set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", properties);
-                    set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", properties);
-
-                    set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", properties);
-                    set_bool_property(obj, WPD_OBJECT_CAN_DELETE, "can_delete", properties);
-                    set_bool_property(obj, WPD_OBJECT_ISSYSTEM, "is_system", properties);
-
-                    set_size_property(obj, WPD_OBJECT_SIZE, "size", properties);
-                    
                     properties->Release(); properties = NULL;
                 }
             } // end for loop
@@ -240,6 +246,9 @@ end:
     return folders;
 }
 
+// }}}
+
+// find_all_objects_in() {{{
 static BOOL find_all_objects_in(IPortableDeviceContent *content, IPortableDevicePropVariantCollection *object_ids, const wchar_t *parent_id) {
     /*
      * Find all children of the object identified by parent_id, recursively.
@@ -286,9 +295,82 @@ end:
     if (children != NULL) children->Release();
     PropVariantClear(&pv);
     return ok;
+} // }}}
+
+// Single get filesystem {{{
+
+static PyObject* get_object_properties(IPortableDeviceProperties *devprops, IPortableDeviceKeyCollection *properties, const wchar_t *object_id) {
+    IPortableDeviceValues *values = NULL;
+    HRESULT hr;
+    PyObject *ans = NULL, *temp = NULL;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = devprops->GetValues(object_id, properties, &values);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get properties for object", hr); goto end; }
+    
+    temp = wchar_to_unicode(object_id);
+    if (temp == NULL) goto end;
+
+    ans = PyDict_New();
+    if (ans == NULL) { PyErr_NoMemory(); goto end; }
+    if (PyDict_SetItemString(ans, "id", temp) != 0) { Py_DECREF(ans); ans = NULL; PyErr_NoMemory(); goto end; }
+
+    set_properties(ans, values);
+     
+end:
+    Py_XDECREF(temp);
+    if (values != NULL) values->Release();
+    return ans;
 }
 
-PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) {
+static PyObject* single_get_filesystem(IPortableDeviceContent *content, const wchar_t *storage_id, IPortableDevicePropVariantCollection *object_ids) {
+    DWORD num, i;
+    PROPVARIANT pv;
+    HRESULT hr;
+    BOOL ok = 1;
+    PyObject *ans = NULL, *item = NULL;
+    IPortableDeviceProperties *devprops = NULL;
+    IPortableDeviceKeyCollection *properties = NULL;
+
+    hr = content->Properties(&devprops);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
+
+    properties = create_filesystem_properties_collection();
+    if (properties == NULL) goto end;
+
+    hr = object_ids->GetCount(&num);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get object id count", hr); goto end; }
+
+    ans = PyDict_New();
+    if (ans == NULL) goto end;
+
+    for (i = 0; i < num; i++) {
+        ok = 0;
+        PropVariantInit(&pv);
+        hr = object_ids->GetAt(i, &pv);
+        if (SUCCEEDED(hr) && pv.pwszVal != NULL) {
+            item = get_object_properties(devprops, properties, pv.pwszVal);
+            if (item != NULL) {
+                PyDict_SetItem(ans, PyDict_GetItemString(item, "id"), item);
+                Py_DECREF(item); item = NULL;
+                ok = 1;
+            }
+        } else hresult_set_exc("Failed to get item from IPortableDevicePropVariantCollection", hr);
+            
+        PropVariantClear(&pv);
+        if (!ok) { Py_DECREF(ans); ans = NULL; break; }
+    }
+
+end:
+    if (devprops != NULL) devprops->Release();
+    if (properties != NULL) properties->Release();
+
+    return ans;
+} 
+// }}}
+
+PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties) { // {{{
     PyObject *folders = NULL;
     IPortableDevicePropVariantCollection *object_ids = NULL;
     IPortableDeviceContent *content = NULL;
@@ -310,12 +392,112 @@ PyObject* wpd::get_filesystem(IPortableDevice *device, const wchar_t *storage_id
     if (!ok) goto end;
 
     if (bulk_properties != NULL) folders = bulk_get_filesystem(device, bulk_properties, storage_id, object_ids);
+    else folders = single_get_filesystem(content, storage_id, object_ids);
 
 end:
     if (content != NULL) content->Release();
     if (object_ids != NULL) object_ids->Release();
 
     return folders;
-}
+} // }}}
+
+PyObject* wpd::get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback) { // {{{
+    IPortableDeviceContent *content = NULL;
+    IPortableDeviceResources *resources = NULL;
+    IPortableDeviceProperties *devprops = NULL;
+    IPortableDeviceValues *values = NULL;
+    IPortableDeviceKeyCollection *properties = NULL;
+    IStream *stream = NULL;
+    HRESULT hr;
+    DWORD bufsize = 4096;
+    char *buf = NULL;
+    ULONG bytes_read = 0, total_read = 0;
+    BOOL ok = FALSE;
+    PyObject *res = NULL;
+    ULONGLONG filesize = 0;
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = device->Content(&content);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create content interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->Properties(&devprops);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get IPortableDeviceProperties interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = CoCreateInstance(CLSID_PortableDeviceKeyCollection, NULL,
+            CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&properties));
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create filesystem properties collection", hr); goto end; }
+    hr = properties->Add(WPD_OBJECT_SIZE);
+    if (FAILED(hr)) { hresult_set_exc("Failed to add filesize property to properties collection", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = devprops->GetValues(object_id, properties, &values);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to get filesize for object", hr); goto end; }
+    hr = values->GetUnsignedLargeIntegerValue(WPD_OBJECT_SIZE, &filesize);
+    if (FAILED(hr)) { hresult_set_exc("Failed to get filesize from values collection", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = content->Transfer(&resources);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { hresult_set_exc("Failed to create resources interface", hr); goto end; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    hr = resources->GetStream(object_id, WPD_RESOURCE_DEFAULT, STGM_READ, &bufsize, &stream);
+    Py_END_ALLOW_THREADS;
+    if (FAILED(hr)) { 
+        if (HRESULT_FROM_WIN32(ERROR_BUSY) == hr) {
+            PyErr_SetString(WPDFileBusy, "Object is in use");
+        } else hresult_set_exc("Failed to create stream interface to read from object", hr); 
+        goto end; 
+    }
+
+    buf = (char *)calloc(bufsize+10, 1);
+    if (buf == NULL) { PyErr_NoMemory(); goto end; }
+
+    while (TRUE) {
+        bytes_read = 0;
+        Py_BEGIN_ALLOW_THREADS;
+        hr = stream->Read(buf, bufsize, &bytes_read);
+        Py_END_ALLOW_THREADS;
+        total_read = total_read + bytes_read;
+        if (hr == STG_E_ACCESSDENIED) { 
+            PyErr_SetString(PyExc_IOError, "Read access is denied to this object"); break; 
+        } else if (hr == S_OK || hr == S_FALSE) {
+            if (bytes_read > 0) {
+                res = PyObject_CallMethod(dest, "write", "s#", buf, bytes_read);
+                if (res == NULL) break;
+                Py_DECREF(res); res = NULL;
+                if (callback != NULL) Py_XDECREF(PyObject_CallFunction(callback, "kK", total_read, filesize));
+            }
+        } else { hresult_set_exc("Failed to read file from device", hr); break; }
+
+        if (hr == S_FALSE || bytes_read < bufsize) { 
+            ok = TRUE; 
+            Py_XDECREF(PyObject_CallMethod(dest, "flush", NULL));
+            break;
+        }
+    }
+
+    if (ok && total_read != filesize) {
+        ok = FALSE;
+        PyErr_SetString(WPDError, "Failed to read all data from file");
+    }
+
+end:
+    if (content != NULL) content->Release();
+    if (devprops != NULL) devprops->Release();
+    if (resources != NULL) resources->Release();
+    if (stream != NULL) stream->Release();
+    if (values != NULL) values->Release();
+    if (properties != NULL) properties->Release();
+    if (buf != NULL) free(buf);
+    if (!ok) return NULL;
+    Py_RETURN_NONE;
+} // }}}
 
 } // namespace wpd

src/calibre/devices/mtp/windows/device.cpp

@@ -78,7 +78,7 @@ update_data(Device *self, PyObject *args, PyObject *kwargs) {
 // get_filesystem() {{{
 static PyObject*
 py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
-    PyObject *storage_id, *ans = NULL;
+    PyObject *storage_id;
     wchar_t *storage;
 
     if (!PyArg_ParseTuple(args, "O", &storage_id)) return NULL;
@@ -88,6 +88,21 @@ py_get_filesystem(Device *self, PyObject *args, PyObject *kwargs) {
     return wpd::get_filesystem(self->device, storage, self->bulk_properties);
 } // }}}
 
+// get_file() {{{
+static PyObject*
+py_get_file(Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *object_id, *stream, *callback = NULL;
+    wchar_t *object;
+
+    if (!PyArg_ParseTuple(args, "OO|O", &object_id, &stream, &callback)) return NULL;
+    object = unicode_to_wchar(object_id);
+    if (object == NULL) return NULL;
+
+    if (callback == NULL || !PyCallable_Check(callback)) callback = NULL;
+
+    return wpd::get_file(self->device, object, stream, callback);
+} // }}}
+
 static PyMethodDef Device_methods[] = {
     {"update_data", (PyCFunction)update_data, METH_VARARGS,
      "update_data() -> Reread the basic device data from the device (total, space, free space, storage locations, etc.)"
@@ -97,6 +112,10 @@ static PyMethodDef Device_methods[] = {
      "get_filesystem(storage_id) -> Get all files/folders on the storage identified by storage_id. Tries to use bulk operations when possible."
     },
 
+    {"get_file", (PyCFunction)py_get_file, METH_VARARGS,
+     "get_file(object_id, stream, callback=None) -> Get the file identified by object_id from the device. The file is written to the stream object, which must be a file like object. If callback is not None, it must be a callable that accepts two arguments: (bytes_read, total_size). It will be called after each chunk is read from the device. Note that it can be called multiple times with the same values."
+    },
+
     {NULL}
 };
 

src/calibre/devices/mtp/windows/driver.py

@@ -7,13 +7,32 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import time
+import time, threading
-from threading import RLock
+from functools import wraps
+from future_builtins import zip
+from itertools import chain
 
 from calibre import as_unicode, prints
 from calibre.constants import plugins, __appname__, numeric_version
+from calibre.ptempfile import SpooledTemporaryFile
 from calibre.devices.errors import OpenFailed
-from calibre.devices.mtp.base import MTPDeviceBase, synchronous
+from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.filesystem_cache import FilesystemCache
+
+class ThreadingViolation(Exception):
+
+    def __init__(self):
+        Exception.__init__('You cannot use the MTP driver from a thread other than the '
+                ' thread in which startup() was called')
+
+def same_thread(func):
+    @wraps(func)
+    def check_thread(self, *args, **kwargs):
+        if self.start_thread is not threading.current_thread():
+            raise ThreadingViolation()
+        return func(self, *args, **kwargs)
+    return check_thread
+
 
 class MTP_DEVICE(MTPDeviceBase):
 
@@ -22,7 +41,6 @@ class MTP_DEVICE(MTPDeviceBase):
     def __init__(self, *args, **kwargs):
         MTPDeviceBase.__init__(self, *args, **kwargs)
         self.dev = None
-        self.lock = RLock()
         self.blacklisted_devices = set()
         self.ejected_devices = set()
         self.currently_connected_pnp_id = None
@@ -31,9 +49,11 @@ class MTP_DEVICE(MTPDeviceBase):
         self.last_refresh_devices_time = time.time()
         self.wpd = self.wpd_error = None
         self._main_id = self._carda_id = self._cardb_id = None
+        self.start_thread = None
+        self._filesystem_cache = None
 
-    @synchronous
     def startup(self):
+        self.start_thread = threading.current_thread()
         self.wpd, self.wpd_error = plugins['wpd']
         if self.wpd is not None:
             try:
@@ -46,13 +66,13 @@ class MTP_DEVICE(MTPDeviceBase):
             except Exception as e:
                 self.wpd_error = as_unicode(e)
 
-    @synchronous
+    @same_thread
     def shutdown(self):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = self.start_thread = None
         if self.wpd is not None:
             self.wpd.uninit()
 
-    @synchronous
+    @same_thread
     def detect_managed_devices(self, devices_on_system):
         if self.wpd is None: return None
 
@@ -119,23 +139,45 @@ class MTP_DEVICE(MTPDeviceBase):
 
         return True
 
-    @synchronous
+    @property
+    def filesystem_cache(self):
+        if self._filesystem_cache is None:
+            ts = self.total_space()
+            all_storage = []
+            items = []
+            for storage_id, capacity in zip([self._main_id, self._carda_id,
+                self._cardb_id], ts):
+                if storage_id is None: continue
+                name = _('Unknown')
+                for s in self.dev.data['storage']:
+                    if s['id'] == storage_id:
+                        name = s['name']
+                        break
+                storage = {'id':storage_id, 'size':capacity, 'name':name,
+                        'is_folder':True}
+                id_map = self.dev.get_filesystem(storage_id)
+                all_storage.append(storage)
+                items.append(id_map.itervalues())
+            self._filesystem_cache = FilesystemCache(all_storage, chain(*items))
+        return self._filesystem_cache
+
+    @same_thread
     def post_yank_cleanup(self):
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
-    @synchronous
+    @same_thread
     def eject(self):
         if self.currently_connected_pnp_id is None: return
         self.ejected_devices.add(self.currently_connected_pnp_id)
         self.currently_connected_pnp_id = self.current_friendly_name = None
         self._main_id = self._carda_id = self._cardb_id = None
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
 
-    @synchronous
+    @same_thread
     def open(self, connected_device, library_uuid):
-        self.dev = self.filesystem_cache = None
+        self.dev = self._filesystem_cache = None
         try:
             self.dev = self.wpd.Device(connected_device)
         except self.wpd.WPDError:
@@ -158,13 +200,13 @@ class MTP_DEVICE(MTPDeviceBase):
             self._cardb_id = storage[2]['id']
         self.current_friendly_name = devdata.get('friendly_name', None)
 
-    @synchronous
+    @same_thread
     def get_device_information(self, end_session=True):
         d = self.dev.data
         dv = d.get('device_version', '')
         return (self.current_friendly_name, dv, dv, '')
 
-    @synchronous
+    @same_thread
     def card_prefix(self, end_session=True):
         ans = [None, None]
         if self._carda_id is not None:
@@ -173,7 +215,7 @@ class MTP_DEVICE(MTPDeviceBase):
             ans[1] = 'mtp:::%s:::'%self._cardb_id
         return tuple(ans)
 
-    @synchronous
+    @same_thread
     def total_space(self, end_session=True):
         ans = [0, 0, 0]
         dd = self.dev.data
@@ -184,7 +226,7 @@ class MTP_DEVICE(MTPDeviceBase):
                 ans[i] = s['capacity']
         return tuple(ans)
 
-    @synchronous
+    @same_thread
     def free_space(self, end_session=True):
         self.dev.update_data()
         ans = [0, 0, 0]
@@ -196,5 +238,14 @@ class MTP_DEVICE(MTPDeviceBase):
                 ans[i] = s['free_space']
         return tuple(ans)
 
-
+    @same_thread
+    def get_file(self, object_id, stream=None, callback=None):
+        if stream is None:
+            stream = SpooledTemporaryFile(5*1024*1024, '_wpd_receive_file.dat')
+        try:
+            self.dev.get_file(object_id, stream, callback)
+        except self.wpd.WPDFileBusy:
+            time.sleep(2)
+            self.dev.get_file(object_id, stream, callback)
+        return stream
 

src/calibre/devices/mtp/windows/global.h

@@ -20,7 +20,7 @@
 namespace wpd {
 
 // Module exception types
-extern PyObject *WPDError, *NoWPD;
+extern PyObject *WPDError, *NoWPD, *WPDFileBusy;
 
 // The global device manager
 extern IPortableDeviceManager *portable_device_manager;
@@ -50,13 +50,14 @@ extern PyTypeObject DeviceType;
 // Utility functions
 PyObject *hresult_set_exc(const char *msg, HRESULT hr);
 wchar_t *unicode_to_wchar(PyObject *o);
-PyObject *wchar_to_unicode(wchar_t *o);
+PyObject *wchar_to_unicode(const wchar_t *o);
 int pump_waiting_messages();
 
 extern IPortableDeviceValues* get_client_information();
 extern IPortableDevice* open_device(const wchar_t *pnp_id, IPortableDeviceValues *client_information);
 extern PyObject* get_device_information(IPortableDevice *device, IPortableDevicePropertiesBulk **bulk_properties);
 extern PyObject* get_filesystem(IPortableDevice *device, const wchar_t *storage_id, IPortableDevicePropertiesBulk *bulk_properties);
+extern PyObject* get_file(IPortableDevice *device, const wchar_t *object_id, PyObject *dest, PyObject *callback);
 
 }
 

src/calibre/devices/mtp/windows/remote.py

@@ -7,8 +7,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import subprocess, sys, os, pprint, signal, time, glob
+import subprocess, sys, os, pprint, signal, time, glob, io
-pprint
+pprint, io
 
 def build(mod='wpd'):
     master = subprocess.Popen('ssh -MN getafix'.split())
@@ -70,7 +70,10 @@ def main():
         print ('Connected to:', dev.get_gui_name())
         print ('Total space', dev.total_space())
         print ('Free space', dev.free_space())
-        pprint.pprint(dev.dev.get_filesystem(dev._main_id))
+        dev.filesystem_cache.dump()
+        # print ('Fetching file: oFF (198214 bytes)')
+        # stream = dev.get_file('oFF')
+        # print ("Fetched size: ", stream.tell())
     finally:
         dev.shutdown()
 

src/calibre/devices/mtp/windows/utils.cpp

@@ -43,7 +43,7 @@ wchar_t *wpd::unicode_to_wchar(PyObject *o) {
     return buf;
 }
 
-PyObject *wpd::wchar_to_unicode(wchar_t *o) {
+PyObject *wpd::wchar_to_unicode(const wchar_t *o) {
     PyObject *ans;
     if (o == NULL) return NULL;
     ans = PyUnicode_FromWideChar(o, wcslen(o));

src/calibre/devices/mtp/windows/wpd.cpp

@@ -10,7 +10,7 @@
 using namespace wpd;
 
 // Module exception types
-PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL;
+PyObject *wpd::WPDError = NULL, *wpd::NoWPD = NULL, *wpd::WPDFileBusy = NULL;
 
 // The global device manager
 IPortableDeviceManager *wpd::portable_device_manager = NULL;
@@ -199,6 +199,9 @@ initwpd(void) {
     NoWPD = PyErr_NewException("wpd.NoWPD", NULL, NULL);
     if (NoWPD == NULL) return;
 
+    WPDFileBusy = PyErr_NewException("wpd.WPDFileBusy", NULL, NULL);
+    if (WPDFileBusy == NULL) return;
+
     Py_INCREF(&DeviceType);
     PyModule_AddObject(m, "Device", (PyObject *)&DeviceType);
 

src/calibre/devices/prst1/driver.py

@@ -50,10 +50,10 @@ class PRST1(USBMS):
 
     VENDOR_NAME        = 'SONY'
     WINDOWS_MAIN_MEM   = re.compile(
-            r'(PRS-T1&)'
+            r'(PRS-T(1|2)&)'
             )
     WINDOWS_CARD_A_MEM = re.compile(
-            r'(PRS-T1__SD&)'
+            r'(PRS-T(1|2)__SD&)'
             )
     MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory'
     STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card'

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -235,7 +235,7 @@ class Serializer(object):
                     itemhref = re.sub(r'article_\d+/', '', itemhref)
                 self.href_offsets[itemhref].append(buf.tell())
                 buf.write('0000000000')
-                buf.write(' ><font size="+1" color="blue"><b><u>')
+                buf.write(' ><font size="+1"><b><u>')
                 t = tocitem.title
                 if isinstance(t, unicode):
                     t = t.encode('utf-8')

src/calibre/ebooks/oeb/display/mathjax.coffee

@@ -23,24 +23,21 @@ class MathJax
         this.pending_cfi = null
         this.hub = null
 
-    load_mathjax: (script) ->
+    load_mathjax: (user_config) ->
         if this.base == null
             log('You must specify the path to the MathJax installation before trying to load MathJax')
             return null
 
-        created = false
+        script = document.createElement('script')
-        if script == null
-            script = document.createElement('script')
-            created = true
 
         script.type = 'text/javascript'
         script.src = 'file://' + this.base + '/MathJax.js'
-
+        script.text = user_config + '''
-        script.text = script.text + '''
+        MathJax.Hub.signal.Interest(function (message) {if (String(message).match(/error/i)) {console.log(message)}});
         MathJax.Hub.Config({
             positionToHash: false,
             showMathMenu: false,
-            extensions: ["tex2jax.js","asciimath2jax.js","mml2jax.js"],
+            extensions: ["tex2jax.js", "asciimath2jax.js", "mml2jax.js"],
             jax: ["input/TeX","input/MathML","input/AsciiMath","output/SVG"],
             TeX: {
                 extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
@@ -50,9 +47,7 @@ class MathJax
         MathJax.Hub.Register.StartupHook("End", window.mathjax.load_finished);
         window.mathjax.hub = MathJax.Hub
         '''
-
+        document.head.appendChild(script)
-        if created
-            document.head.appendChild(script)
 
     load_finished: () =>
         log('MathJax load finished!')
@@ -67,14 +62,17 @@ class MathJax
         this.math_present = false
         this.math_loaded = false
         this.pending_cfi = null
+        user_config = ''
         for c in document.getElementsByTagName('script')
             if c.getAttribute('type') == 'text/x-mathjax-config'
+                if c.text
+                    user_config += c.text
                 script = c
-                break
+                c.parentNode.removeChild(c)
 
         if script != null or document.getElementsByTagName('math').length > 0
             this.math_present = true
-            this.load_mathjax(script)
+            this.load_mathjax(user_config)
 
     after_resize: () ->
         if not this.math_present or this.hub == null

src/calibre/ebooks/oeb/display/paged.coffee

@@ -334,6 +334,16 @@ class PagedDisplay
                 elem = elems[0]
         if not elem
             return
+        if window.mathjax?.math_present
+            # MathJax links to children of SVG tags and scrollIntoView doesn't
+            # work properly for them, so if this link points to something
+            # inside an <svg> tag we instead scroll the parent of the svg tag
+            # into view.
+            parent = elem
+            while parent and parent?.tagName?.toLowerCase() != 'svg'
+                parent = parent.parentNode
+            if parent?.tagName?.toLowerCase() == 'svg'
+                elem = parent.parentNode
         elem.scrollIntoView()
         if this.in_paged_mode
             # Ensure we are scrolled to the column containing elem
@@ -368,7 +378,9 @@ class PagedDisplay
         # The Conformal Fragment Identifier at the current position, returns
         # null if it could not be calculated. Requires the cfi.coffee library.
         ans = null
-        if not window.cfi?
+        if not window.cfi? or (window.mathjax?.math_present and not window.mathjax?.math_loaded)
+            # If MathJax is loading, it is changing the DOM, so we cannot
+            # reliably generate a CFI
             return ans
         if this.in_paged_mode
             c = this.current_column_location()
@@ -402,9 +414,9 @@ class PagedDisplay
         return ans
 
     click_for_page_turn: (event) ->
-        # Check if the click event event should generate a apge turn. Returns
+        # Check if the click event should generate a page turn. Returns
         # null if it should not, true if it is a backwards page turn, false if
-        # it is a forward apge turn.
+        # it is a forward page turn.
         left_boundary = this.current_margin_side
         right_bondary = this.screen_width - this.current_margin_side
         if left_boundary > event.clientX

src/calibre/gui2/store/stores/bn_plugin.py

@@ -34,26 +34,29 @@ class BNStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook' % (query.replace(' ', '-'), urllib.quote_plus(query))
+        url = 'http://www.barnesandnoble.com/s/%s?keyword=%s&store=ebook&view=list' % (query.replace(' ', '-'), urllib.quote_plus(query))
 
         br = browser()
 
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
+            raw = f.read()
+            doc = html.fromstring(raw)
             for data in doc.xpath('//ul[contains(@class, "result-set")]/li[contains(@class, "result")]'):
                 if counter <= 0:
                     break
 
-                id = ''.join(data.xpath('.//div[contains(@class, "image-bounding-box")]/a/@href'))
+                id = ''.join(data.xpath('.//div[contains(@class, "image-block")]/a/@href'))
                 if not id:
                     continue
 
                 cover_url = ''.join(data.xpath('.//img[contains(@class, "product-image")]/@src'))
 
-                title = ''.join(data.xpath('.//a[@class="title"]//text()'))
+                title = ''.join(data.xpath('descendant::p[@class="title"]//span[@class="name"]//text()')).strip()
-                author = ', '.join(data.xpath('.//a[@class="contributor"]//text()'))
+                if not title: continue
-                price = ''.join(data.xpath('.//div[@class="price-format"]//span[contains(@class, "price")]/text()'))
+
+                author = ', '.join(data.xpath('.//ul[@class="contributors"]//a[@class="subtle"]//text()')).strip()
+                price = ''.join(data.xpath('.//a[contains(@class, "bn-price")]//text()'))
 
                 counter -= 1
 

src/calibre/gui2/viewer/main.py

@@ -185,6 +185,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.pos.setDecimals(1)
         self.pos.setSuffix('/'+_('Unknown')+'     ')
         self.pos.setMinimum(1.)
+        self.splitter.setCollapsible(0, False)
+        self.splitter.setCollapsible(1, False)
         self.pos.setMinimumWidth(150)
         self.tool_bar2.insertWidget(self.action_find_next, self.pos)
         self.reference = Reference()
@@ -1028,6 +1030,8 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         av = available_height() - 30
         if self.height() > av:
             self.resize(self.width(), av)
+        self.splitter.setCollapsible(0, False)
+        self.splitter.setCollapsible(1, False)
 
 def config(defaults=None):
     desc = _('Options to control the ebook viewer')

src/calibre/gui2/viewer/main.ui

@@ -27,7 +27,14 @@
       <property name="childrenCollapsible">
        <bool>false</bool>
       </property>
-      <widget class="TOCView" name="toc"/>
+      <widget class="TOCView" name="toc">
+       <property name="minimumSize">
+        <size>
+         <width>150</width>
+         <height>0</height>
+        </size>
+       </property>
+      </widget>
       <widget class="QFrame" name="frame">
        <property name="frameShape">
         <enum>QFrame::StyledPanel</enum>