0.8.63+

manual/Makefile

@@ -60,7 +60,7 @@ htmlhelp:
 
 latex:
 	mkdir -p .build/latex .build/doctrees
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex
+	$(SPHINXBUILD) -b mylatex $(ALLSPHINXOPTS) .build/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in .build/latex."
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \

manual/conf.py

@@ -14,10 +14,10 @@
 import sys, os
 
 # If your extensions are in another directory, add it here.
-sys.path.append(os.path.abspath('../src'))
 sys.path.append(os.path.abspath('.'))
-__appname__ = os.environ.get('__appname__', 'calibre')
+import init_calibre
-__version__ = os.environ.get('__version__', '0.0.0')
+init_calibre
+from calibre.constants import __appname__, __version__
 import custom
 custom
 # General configuration
@@ -154,7 +154,8 @@ latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
-#latex_documents = []
+latex_documents = [('index', 'calibre.tex', 'calibre User Manual',
+    'Kovid Goyal', 'manual', False)]
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
@@ -164,3 +165,11 @@ latex_font_size = '10pt'
 
 # If false, no module index is generated.
 #latex_use_modindex = True
+
+latex_logo = 'resources/logo.png'
+latex_show_pagerefs = True
+latex_show_urls = 'footnote'
+latex_elements = {
+'papersize':'letterpaper',
+'fontenc':r'\usepackage[T2A,T1]{fontenc}'
+}

manual/custom.py

@@ -14,6 +14,7 @@ from sphinx.util.console import bold
 sys.path.append(os.path.abspath('../../../'))
 from calibre.linux import entry_points
 from epub import EPUBHelpBuilder
+from latex import LaTeXHelpBuilder
 
 def substitute(app, doctree):
     pass
@@ -251,6 +252,7 @@ def template_docs(app):
 def setup(app):
     app.add_config_value('kovid_epub_cover', None, False)
     app.add_builder(EPUBHelpBuilder)
+    app.add_builder(LaTeXHelpBuilder)
     app.connect('doctree-read', substitute)
     app.connect('builder-inited', generate_docs)
     app.connect('build-finished', finished)

manual/index.rst

@@ -17,7 +17,7 @@ To get started with more advanced usage, you should read about the :ref:`Graphic
 
 .. only:: online
     
-    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_ and `AZW3 (Kindle Fire) format <calibre.azw3>`_.
+    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_,  `AZW3 (Kindle Fire) format <calibre.azw3>`_ and `PDF format <calibre.pdf>`_.
     
 Sections
 ------------

manual/latex.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+
+from sphinx.builders.latex import LaTeXBuilder
+
+class LaTeXHelpBuilder(LaTeXBuilder):
+    name = 'mylatex'
+
+    def finish(self):
+        LaTeXBuilder.finish(self)
+        self.info('Fixing Cyrillic characters...')
+        tex = os.path.join(self.outdir, 'calibre.tex')
+        with open(tex, 'r+b') as f:
+            raw = f.read().replace(b'Михаил Горбачёв',
+                br'{\fontencoding{T2A}\selectfont Михаил Горбачёв}')
+            f.seek(0)
+            f.write(raw)

recipes/ekundelek_pl.recipe

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+__license__ = 'GPL v3'
+__copyright__ = u'2012, Artur Stachecki <artur.stachecki@gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class swiatczytnikow(BasicNewsRecipe):
+    title = u'eKundelek'
+    description = u'Najsympatyczniejszy blog o e-czytnikach Kindle'
+    language = 'pl'
+    __author__ = u'Artur Stachecki'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    remove_tags = [dict(name = 'div', attrs = {'class' : 'feedflare'})]
+
+    feeds = [(u'Wpisy', u'http://feeds.feedburner.com/Ekundelekpl?format=xml')]

recipes/metro_uk.recipe

@@ -1,31 +1,42 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1306097511(BasicNewsRecipe):
     title          = u'Metro UK'
-    description = 'News as provide by The Metro -UK'
+    description = 'Author Dave Asbury : News as provide by The Metro -UK'
     #timefmt = ''
     __author__ = 'Dave Asbury'
-    #last update 9/6/12
+    #last update 4/8/12
     cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg'
-    #no_stylesheets = True
+    no_stylesheets = True
     oldest_article = 1
-    max_articles_per_feed = 10
+    max_articles_per_feed = 12
     remove_empty_feeds = True
     remove_javascript     = True
-    auto_cleanup = True
+    #auto_cleanup = True
     encoding = 'UTF-8'
-
+    cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/157897_117118184990145_840702264_n.jpg'
     language = 'en_GB'
     masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:1.6em;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:1.2em;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:1.0em;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:1.0em;}
+		'''
     keep_only_tags = [
-
+		#dict(name='h1'),
-    ]
+		#dict(name='h2'),
+		#dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']})
+	#dict(name='h3'),
+	#dict(attrs={'class' :  'BText'}),
+		]
     remove_tags    = [
-
+                                dict(name='span',attrs={'class' : 'share'}),
+		dict(name='li'),
+		dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}),
+           		dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']})
                                ]
-
+    remove_tags_before = dict(name='h1')
+    #remove_tags_after = dict(attrs={'id':['topic-buttons']})
 
     feeds          = [
         (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
-    extra_css  = '''
-	body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
-                	 '''

recipes/sueddeutsche_mobil.recipe

@@ -0,0 +1,117 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+__license__   = 'GPL v3'
+__copyright__ = '2012, Andreas Zeiser <andreas.zeiser@web.de>'
+'''
+szmobil.sueddeutsche.de/
+'''
+
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+import re
+
+class SZmobil(BasicNewsRecipe):
+    title       = u'Süddeutsche Zeitung mobil'
+    __author__  = u'Andreas Zeiser'
+    description = u'Nachrichten aus Deutschland. Zugriff auf kostenpflichtiges Abo SZ mobil.'
+    publisher              = u'Sueddeutsche Zeitung'
+    language               = u'de'
+    publication_type       = u'newspaper'
+    category               = u'news, politics, Germany'
+
+    no_stylesheets         = True
+    oldest_article         = 2
+    encoding               = 'iso-8859-1'
+    needs_subscription     = True
+    remove_empty_feeds     = True
+    delay                  = 1
+    cover_source           = 'http://www.sueddeutsche.de/verlag'
+
+    timefmt = ' [%a, %d %b, %Y]'
+
+    root_url ='http://szmobil.sueddeutsche.de/'
+    keep_only_tags = [dict(name='div', attrs={'class':'article'})]
+
+    def get_cover_url(self):
+        src = self.index_to_soup(self.cover_source)
+        image_url = src.find(attrs={'class':'preview-image'})
+        return image_url.div.img['src']
+
+    def get_browser(self):
+        browser = BasicNewsRecipe.get_browser(self)
+
+        # Login via fetching of Streiflicht -> Fill out login request
+        url = self.root_url + 'show.php?id=streif'
+        browser.open(url)
+
+        browser.select_form(nr=0) # to select the first form
+        browser['username'] = self.username
+        browser['password'] = self.password
+        browser.submit()
+
+        return browser
+
+	def parse_index(self):
+        # find all sections
+        src = self.index_to_soup('http://szmobil.sueddeutsche.de')
+        feeds = []
+        for itt in src.findAll('a',href=True):
+            if itt['href'].startswith('show.php?section'):
+                feeds.append( (itt.string[0:-2],itt['href']) )
+
+        all_articles = []
+        for feed in feeds:
+            feed_url = self.root_url + feed[1]
+            feed_title = feed[0]
+
+            self.report_progress(0, ('Fetching feed')+' %s...'%(feed_title if feed_title else feed_url))
+
+            src = self.index_to_soup(feed_url)
+            articles = []
+            shorttitles = dict()
+            for itt in src.findAll('a', href=True):
+                if itt['href'].startswith('show.php?id='):
+                    article_url = itt['href']
+                    article_id = int(re.search("id=(\d*)&etag=", itt['href']).group(1))
+
+                    # first check if link is a special article in section "Meinungsseite"
+                    if itt.find('strong')!= None:
+                        article_name = itt.strong.string
+                        article_shorttitle = itt.contents[1]
+
+                        articles.append( (article_name, article_url, article_id) )
+                        shorttitles[article_id] = article_shorttitle
+                        continue
+
+
+                    # candidate for a general article
+                    if itt.string == None:
+                        article_name = ''
+                    else:
+                        article_name = itt.string
+
+                    if (article_name[0:10] == "&nbsp;mehr"):
+                        # just another link ("mehr") to an article
+                        continue
+
+                    if itt.has_key('id'):
+                        shorttitles[article_id] = article_name
+                    else:
+                        articles.append( (article_name, article_url, article_id) )
+
+            feed_articles = []
+            for article_name, article_url, article_id in articles:
+                url = self.root_url + article_url
+                title = article_name
+                pubdate = strftime('%a, %d %b')
+                description = ''
+                if shorttitles.has_key(article_id):
+                    description = shorttitles[article_id]
+                # we do not want the flag ("Impressum")
+                if "HERAUSGEGEBEN VOM" in description:
+                    continue
+                d = dict(title=title, url=url, date=pubdate, description=description, content='')
+                feed_articles.append(d)
+            all_articles.append( (feed_title, feed_articles) )
+
+        return all_articles
+

setup/extensions.py

@@ -174,6 +174,20 @@ if isosx:
                 ldflags=['-framework', 'IOKit'])
             )
 
+if islinux:
+    extensions.append(Extension('libmtp',
+        [
+        'calibre/devices/mtp/unix/devices.c',
+        'calibre/devices/mtp/unix/libmtp.c'
+        ],
+        headers=[
+        'calibre/devices/mtp/unix/devices.h',
+        'calibre/devices/mtp/unix/upstream/music-players.h',
+        'calibre/devices/mtp/unix/upstream/device-flags.h',
+        ],
+        libraries=['mtp']
+    ))
+
 if isunix:
     cc = os.environ.get('CC', 'gcc')
     cxx = os.environ.get('CXX', 'g++')

setup/publish.py

@@ -80,8 +80,17 @@ class Manual(Command):
                                    '-d', '.build/doctrees', '.', '.build/html'])
             subprocess.check_call(['sphinx-build', '-b', 'myepub', '-d',
                                    '.build/doctrees', '.', '.build/epub'])
+            subprocess.check_call(['sphinx-build', '-b', 'mylatex', '-d',
+                                   '.build/doctrees', '.', '.build/latex'])
+            pwd = os.getcwdu()
+            os.chdir('.build/latex')
+            subprocess.check_call(['make', 'all-pdf'], stdout=open(os.devnull,
+                'wb'))
+            os.chdir(pwd)
             epub_dest = self.j('.build', 'html', 'calibre.epub')
+            pdf_dest = self.j('.build', 'html', 'calibre.pdf')
             shutil.copyfile(self.j('.build', 'epub', 'calibre.epub'), epub_dest)
+            shutil.copyfile(self.j('.build', 'latex', 'calibre.pdf'), pdf_dest)
             subprocess.check_call(['ebook-convert', epub_dest,
                 epub_dest.rpartition('.')[0] + '.azw3',
                 '--page-breaks-before=/', '--disable-font-rescaling',

src/calibre/constants.py

@@ -93,6 +93,8 @@ class Plugins(collections.Mapping):
             plugins.append('winutil')
         if isosx:
             plugins.append('usbobserver')
+        if islinux:
+            plugins.append('libmtp')
         self.plugins = frozenset(plugins)
 
     def load_plugin(self, name):

src/calibre/devices/android/driver.py

@@ -43,6 +43,7 @@ class ANDROID(USBMS):
                        0xccf  : HTC_BCDS,
                        0xcd6  : HTC_BCDS,
                        0xce5  : HTC_BCDS,
+                       0xcec  : HTC_BCDS,
                        0x2910 : HTC_BCDS,
                        0xff9  : HTC_BCDS,
             },

src/calibre/devices/interface.py

@@ -199,7 +199,7 @@ class DevicePlugin(Plugin):
     # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
-            detected_device=None) :
+            detected_device=None):
         """
         :param key: The key to unlock the device
         :param log_packets: If true the packet stream to/from the device is logged
@@ -297,7 +297,7 @@ class DevicePlugin(Plugin):
 
         :return: (device name, device version, software version on device, mime type)
                  The tuple can optionally have a fifth element, which is a
-                 drive information diction. See usbms.driver for an example.
+                 drive information dictionary. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -607,7 +607,7 @@ class BookList(list):
         pass
 
     def supports_collections(self):
-        ''' Return True if the the device supports collections for this book list. '''
+        ''' Return True if the device supports collections for this book list. '''
         raise NotImplementedError()
 
     def add_book(self, book, replace_metadata):

src/calibre/devices/mtp/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/devices/mtp/base.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.devices.interface import DevicePlugin
+
+class MTPDeviceBase(DevicePlugin):
+    name = 'SmartDevice App Interface'
+    gui_name = _('MTP Device')
+    icon = I('devices/galaxy_s3.png')
+    description = _('Communicate with MTP devices')
+    author = 'Kovid Goyal'
+    version = (1, 0, 0)
+
+    # Invalid USB vendor information so the scanner will never match
+    VENDOR_ID                   = [0xffff]
+    PRODUCT_ID                  = [0xffff]
+    BCD                         = [0xffff]
+
+    THUMBNAIL_HEIGHT = 128
+    CAN_SET_METADATA = []
+
+    BACKLOADING_ERROR_MESSAGE = None
+
+    def __init__(self, *args, **kwargs):
+        DevicePlugin.__init__(self, *args, **kwargs)
+        self.progress_reporter = None
+
+    def reset(self, key='-1', log_packets=False, report_progress=None,
+            detected_device=None):
+        pass
+
+    def set_progress_reporter(self, report_progress):
+        self.progress_reporter = report_progress
+

src/calibre/devices/mtp/unix/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+'''
+libmtp based drivers for MTP devices on Unix like platforms.
+'''
+

src/calibre/devices/mtp/unix/detect.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.constants import plugins
+
+class MTPDetect(object):
+
+    def __init__(self):
+        p = plugins['libmtp']
+        self.libmtp = p[0]
+        if self.libmtp is None:
+            print ('Failed to load libmtp, MTP device detection disabled')
+            print (p[1])
+        self.cache = {}
+
+    def __call__(self, devices):
+        '''
+        Given a list of devices as returned by LinuxScanner, return the set of
+        devices that are likely to be MTP devices. This class maintains a cache
+        to minimize USB polling. Note that detection is partially based on a
+        list of known vendor and product ids. This is because polling some
+        older devices causes problems. Therefore, if this method identifies a
+        device as MTP, it is not actually guaranteed that it will be a working
+        MTP device.
+        '''
+        # First drop devices that have been disconnected from the cache
+        connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
+            d.bcd, d.serial) for d in devices}
+        for d in tuple(self.cache.iterkeys()):
+            if d not in connected_devices:
+                del self.cache[d]
+
+        # Since is_mtp_device() can cause USB traffic by probing the device, we
+        # cache its result
+        mtp_devices = set()
+        if self.libmtp is None:
+            return mtp_devices
+
+        for d in devices:
+            ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial), None)
+            if ans is None:
+                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
+                        d.vendor_id, d.product_id)
+                self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial)] = ans
+            if ans:
+                mtp_devices.add(d)
+        return mtp_devices
+
+    def create_device(self, connected_device):
+        d = connected_device
+        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.manufacturer, d.product, d.serial)
+
+if __name__ == '__main__':
+    from calibre.devices.scanner import linux_scanner
+    mtp_detect = MTPDetect()
+    devs = mtp_detect(linux_scanner())
+    print ('Found %d MTP devices:'%len(devs))
+    for dev in devs:
+        print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
+        print()
+
+

src/calibre/devices/mtp/unix/devices.c

@@ -0,0 +1,16 @@
+/*
+ * devices.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+#include "upstream/device-flags.h"
+#include "devices.h"
+
+const calibre_device_entry_t calibre_mtp_device_table[] = {
+#include "upstream/music-players.h"
+
+  , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
+};
+

src/calibre/devices/mtp/unix/devices.h

@@ -0,0 +1,22 @@
+#pragma once
+/*
+ * devices.h
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+#include <stdint.h>
+#include <stddef.h>
+
+struct calibre_device_entry_struct {
+  char *vendor; /**< The vendor of this device */
+  uint16_t vendor_id; /**< Vendor ID for this device */
+  char *product; /**< The product name of this device */
+  uint16_t product_id; /**< Product ID for this device */
+  uint32_t device_flags; /**< Bugs, device specifics etc */
+};
+
+typedef struct calibre_device_entry_struct calibre_device_entry_t;
+
+extern const calibre_device_entry_t calibre_mtp_device_table[];
+

src/calibre/devices/mtp/unix/driver.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time, operator
+from threading import RLock
+from functools import wraps
+
+from calibre.devices.errors import OpenFailed
+from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.unix.detect import MTPDetect
+
+def synchronous(func):
+    @wraps(func)
+    def synchronizer(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return synchronizer
+
+class MTP_DEVICE(MTPDeviceBase):
+
+    supported_platforms = ['linux']
+
+    def __init__(self, *args, **kwargs):
+        MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.detect = MTPDetect()
+        self.dev = None
+        self.lock = RLock()
+        self.blacklisted_devices = set()
+
+    def report_progress(self, sent, total):
+        try:
+            p = int(sent/total * 100)
+        except ZeroDivisionError:
+            p = 100
+        if self.progress_reporter is not None:
+            self.progress_reporter(p)
+
+    @synchronous
+    def get_gui_name(self):
+        if self.dev is None or not self.dev.friendly_name: return self.name
+        return self.dev.friendly_name
+
+    @synchronous
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+
+        # First remove blacklisted devices.
+        devs = []
+        for d in devices_on_system:
+            if (d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
+                devs.append(d)
+
+        devs = self.detect(devs)
+        if self.dev is not None:
+            # Check if the currently opened device is still connected
+            ids = self.dev.ids
+            found = False
+            for d in devs:
+                if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
+                        == ids ):
+                    found = True
+                    break
+            return found
+        # Check if any MTP capable device is present
+        return len(devs) > 0
+
+    @synchronous
+    def post_yank_cleanup(self):
+        self.dev = None
+
+    @synchronous
+    def shutdown(self):
+        self.dev = None
+
+    @synchronous
+    def open(self, connected_device, library_uuid):
+        def blacklist_device():
+            d = connected_device
+            self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
+                    d.product_id, d.bcd, d.serial))
+        try:
+            self.dev = self.detect.create_device(connected_device)
+        except ValueError:
+            # Give the device some time to settle
+            time.sleep(2)
+            try:
+                self.dev = self.detect.create_device(connected_device)
+            except ValueError:
+                # Black list this device so that it is ignored for the
+                # remainder of this session.
+                blacklist_device()
+                raise OpenFailed('%s is not a MTP device'%(connected_device,))
+        except TypeError:
+            blacklist_device()
+            raise OpenFailed('')
+
+        storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
+        if not storage:
+            blacklist_device()
+            raise OpenFailed('No storage found for device %s'%(connected_device,))
+        self._main_id = storage[0]['id']
+        self._carda_id = self._cardb_id = None
+        if len(storage) > 1:
+            self._carda_id = storage[1]['id']
+        if len(storage) > 2:
+            self._cardb_id = storage[2]['id']
+
+    @synchronous
+    def get_device_information(self, end_session=True):
+        d = self.dev
+        return (d.friendly_name, d.device_version, d.device_version, '')
+
+    @synchronous
+    def card_prefix(self, end_session=True):
+        ans = [None, None]
+        if self._carda_id is not None:
+            ans[0] = 'mtp:%d:'%self._carda_id
+        if self._cardb_id is not None:
+            ans[1] = 'mtp:%d:'%self._cardb_id
+        return tuple(ans)
+
+    @synchronous
+    def total_space(self, end_session=True):
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['capacity']
+        return tuple(ans)
+
+    @synchronous
+    def free_space(self, end_session=True):
+        self.dev.update_storage_info()
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['freespace_bytes']
+        return tuple(ans)
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+    dev = MTP_DEVICE(None)
+    from calibre.devices.scanner import linux_scanner
+    devs = linux_scanner()
+    mtp_devs = dev.detect(devs)
+    dev.open(list(mtp_devs)[0], 'xxx')
+    d = dev.dev
+    print ("Opened device:", dev.get_gui_name())
+    print ("Storage info:")
+    pprint(d.storage_info)
+    print("Free space:", dev.free_space())
+    files, errs = d.get_filelist(dev)
+    pprint((len(files), errs))
+    folders, errs = d.get_folderlist()
+    pprint((len(folders), errs))
+

src/calibre/devices/mtp/unix/libmtp.c

@@ -0,0 +1,559 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+#include <libmtp.h>
+
+#include "devices.h"
+
+// Macros and utilities
+#define ENSURE_DEV(rval) \
+    if (self->device == NULL) { \
+        PyErr_SetString(PyExc_ValueError, "This device has not been initialized."); \
+        return rval; \
+    }
+
+#define ENSURE_STORAGE(rval) \
+    if (self->device->storage == NULL) { \
+        PyErr_SetString(PyExc_RuntimeError, "The device has no storage information."); \
+        return rval; \
+    }
+
+// Storage types
+#define ST_Undefined            0x0000
+#define ST_FixedROM             0x0001
+#define ST_RemovableROM         0x0002
+#define ST_FixedRAM             0x0003
+#define ST_RemovableRAM         0x0004
+
+// Storage Access capability
+#define AC_ReadWrite            0x0000
+#define AC_ReadOnly             0x0001
+#define AC_ReadOnly_with_Object_Deletion    0x0002
+
+typedef struct {
+    PyObject *obj;
+    PyThreadState *state;
+} ProgressCallback;
+
+static int report_progress(uint64_t const sent, uint64_t const total, void const *const data) {
+    PyObject *res;
+    ProgressCallback *cb;
+
+    cb = (ProgressCallback *)data;
+    if (cb->obj != NULL) {
+        PyEval_RestoreThread(cb->state);
+        res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
+        Py_XDECREF(res);
+        cb->state = PyEval_SaveThread();
+    }
+    return 0;
+}
+
+static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
+    LIBMTP_error_t *stack;
+    PyObject *err;
+
+    for(stack = LIBMTP_Get_Errorstack(dev); stack != NULL; stack=stack->next) {
+        err = Py_BuildValue("Is", stack->errornumber, stack->error_text);
+        if (err == NULL) break;
+        PyList_Append(list, err);
+        Py_DECREF(err);
+    }
+
+    LIBMTP_Clear_Errorstack(dev);
+}
+
+// }}}
+
+// Device object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    LIBMTP_mtpdevice_t *device;
+    PyObject *ids;
+    PyObject *friendly_name;
+    PyObject *manufacturer_name;
+    PyObject *model_name;
+    PyObject *serial_number;
+    PyObject *device_version;
+
+} libmtp_Device;
+
+// Device.__init__() {{{
+static void
+libmtp_Device_dealloc(libmtp_Device* self)
+{
+    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    self->device = NULL;
+
+    Py_XDECREF(self->ids); self->ids = NULL;
+    Py_XDECREF(self->friendly_name); self->friendly_name = NULL;
+    Py_XDECREF(self->manufacturer_name); self->manufacturer_name = NULL;
+    Py_XDECREF(self->model_name); self->model_name = NULL;
+    Py_XDECREF(self->serial_number); self->serial_number = NULL;
+    Py_XDECREF(self->device_version); self->device_version = NULL;
+
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static int
+libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
+{
+    int busnum, devnum, vendor_id, product_id;
+    PyObject *usb_serialnum;
+    char *vendor, *product, *friendly_name, *manufacturer_name, *model_name, *serial_number, *device_version;
+    LIBMTP_raw_device_t rawdev;
+    LIBMTP_mtpdevice_t *dev;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiiissO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
+
+    if (devnum < 0 || devnum > 255 || busnum < 0) { PyErr_SetString(PyExc_TypeError, "Invalid busnum/devnum"); return -1; }
+
+    self->ids = Py_BuildValue("iiiiO", busnum, devnum, vendor_id, product_id, usb_serialnum);
+    if (self->ids == NULL) return -1;
+
+    rawdev.bus_location = (uint32_t)busnum;
+    rawdev.devnum = (uint8_t)devnum;
+    rawdev.device_entry.vendor = vendor;
+    rawdev.device_entry.product = product;
+    rawdev.device_entry.vendor_id = vendor_id;
+    rawdev.device_entry.product_id = product_id;
+    rawdev.device_entry.device_flags = 0x00000000U;
+
+    Py_BEGIN_ALLOW_THREADS;
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == product_id) {
+            rawdev.device_entry.device_flags = calibre_mtp_device_table[i].device_flags;
+        }
+    }
+
+    // Note that contrary to what the libmtp docs imply, we cannot use
+    // LIBMTP_Open_Raw_Device_Uncached as using it causes file listing to fail
+    dev = LIBMTP_Open_Raw_Device(&rawdev);
+    Py_END_ALLOW_THREADS;
+
+    if (dev == NULL) { 
+        PyErr_SetString(PyExc_ValueError, "Unable to open raw device."); 
+        return -1;
+    }
+
+    self->device = dev;
+
+    Py_BEGIN_ALLOW_THREADS;
+    friendly_name = LIBMTP_Get_Friendlyname(self->device);
+    manufacturer_name = LIBMTP_Get_Manufacturername(self->device);
+    model_name = LIBMTP_Get_Modelname(self->device);
+    serial_number = LIBMTP_Get_Serialnumber(self->device);
+    device_version = LIBMTP_Get_Deviceversion(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (friendly_name != NULL) {
+        self->friendly_name = PyUnicode_FromString(friendly_name);
+        free(friendly_name);
+    }
+    if (self->friendly_name == NULL) { self->friendly_name = Py_None; Py_INCREF(Py_None); }
+
+    if (manufacturer_name != NULL) {
+        self->manufacturer_name = PyUnicode_FromString(manufacturer_name);
+        free(manufacturer_name);
+    }
+    if (self->manufacturer_name == NULL) { self->manufacturer_name = Py_None; Py_INCREF(Py_None); }
+
+    if (model_name != NULL) {
+        self->model_name = PyUnicode_FromString(model_name);
+        free(model_name);
+    }
+    if (self->model_name == NULL) { self->model_name = Py_None; Py_INCREF(Py_None); }
+
+    if (serial_number != NULL) {
+        self->serial_number = PyUnicode_FromString(serial_number);
+        free(serial_number);
+    }
+    if (self->serial_number == NULL) { self->serial_number = Py_None; Py_INCREF(Py_None); }
+
+    if (device_version != NULL) {
+        self->device_version = PyUnicode_FromString(device_version);
+        free(device_version);
+    }
+    if (self->device_version == NULL) { self->device_version = Py_None; Py_INCREF(Py_None); }
+
+    return 0;
+}
+// }}}
+
+// Device.friendly_name {{{
+static PyObject *
+libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->friendly_name); return self->friendly_name;
+} // }}}
+
+// Device.manufacturer_name {{{
+static PyObject *
+libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->manufacturer_name); return self->manufacturer_name;
+} // }}}
+
+// Device.model_name {{{
+static PyObject *
+libmtp_Device_model_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->model_name); return self->model_name;
+} // }}}
+
+// Device.serial_number {{{
+static PyObject *
+libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->serial_number); return self->serial_number;
+} // }}}
+
+// Device.device_version {{{
+static PyObject *
+libmtp_Device_device_version(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->device_version); return self->device_version;
+} // }}}
+
+// Device.ids {{{
+static PyObject *
+libmtp_Device_ids(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->ids); return self->ids;
+} // }}}
+
+// Device.update_storage_info() {{{
+static PyObject*
+libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    ENSURE_DEV(NULL);
+    if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to get storage infor for device.");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+// }}}
+
+// Device.storage_info {{{
+static PyObject *
+libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
+    PyObject *ans, *loc;
+    LIBMTP_devicestorage_t *storage;
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    if (ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    for (storage = self->device->storage; storage != NULL; storage = storage->next) {
+        // Ignore read only storage
+        if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM) continue;
+        // Storage IDs with the lower 16 bits 0x0000 are not supposed to be
+        // writeable.
+        if ((storage->id & 0x0000FFFFU) == 0x00000000U) continue;
+        // Also check the access capability to avoid e.g. deletable only storages
+        if (storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) continue;
+
+        loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s}", 
+                "id", storage->id, 
+                "removable", ((storage->StorageType == ST_RemovableRAM) ? Py_True : Py_False),
+                "capacity", storage->MaxCapacity,
+                "freespace_bytes", storage->FreeSpaceInBytes,
+                "freespace_objects", storage->FreeSpaceInObjects,
+                "storage_desc", storage->StorageDescription,
+                "volume_id", storage->VolumeIdentifier
+        );
+
+        if (loc == NULL) return NULL; 
+        if (PyList_Append(ans, loc) != 0) return NULL;
+        Py_DECREF(loc);
+
+    }
+
+    return ans;
+} // }}}
+
+// Device.get_filelist {{{
+static PyObject *
+libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *fo, *callback = NULL, *errs;
+    ProgressCallback cb;
+    LIBMTP_file_t *f, *tf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
+    cb.obj = callback;
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.state = PyEval_SaveThread();
+    tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+
+    if (tf == NULL) { 
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    for (f=tf; f != NULL; f=f->next) {
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                "id", f->item_id,
+                "parent_id", f->parent_id,
+                "storage_id", f->storage_id,
+                "filename", f->filename,
+                "size", f->filesize,
+                "modtime", f->modificationdate
+        );
+        if (fo == NULL || PyList_Append(ans, fo) != 0) break;
+        Py_DECREF(fo);
+    }
+
+    // Release memory
+    f = tf;
+    while (f != NULL) {
+        tf = f; f = f->next; LIBMTP_destroy_file_t(tf);
+    }
+
+    if (callback != NULL) {
+        // Bug in libmtp where it does not call callback with 100%
+        fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
+        Py_XDECREF(fo);
+    }
+
+    return Py_BuildValue("NN", ans, errs);
+} // }}}
+
+// Device.get_folderlist {{{
+
+int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
+    PyObject *folder, *children;
+
+    children = PyList_New(0);
+    if (children == NULL) { PyErr_NoMemory(); return 1;}
+
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+            "id", f->folder_id,
+            "parent_d", f->parent_id,
+            "storage_id", f->storage_id,
+            "name", f->name,
+            "children", children);
+    if (folder == NULL) return 1;
+    PyList_Append(parent, folder);
+    Py_DECREF(folder);
+
+    if (f->sibling != NULL) {
+        if (folderiter(f->sibling, parent)) return 1;
+    }
+
+    if (f->child != NULL) {
+        if (folderiter(f->child, children)) return 1;
+    }
+
+    return 0;
+}
+
+static PyObject *
+libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *errs;
+    LIBMTP_folder_t *f;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (errs == NULL || ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    f = LIBMTP_Get_Folder_List(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (f == NULL) {
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    if (folderiter(f, ans)) return NULL;
+    LIBMTP_destroy_folder_t(f);
+
+    return Py_BuildValue("NN", ans, errs);
+
+} // }}}
+
+static PyMethodDef libmtp_Device_methods[] = {
+    {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
+     "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
+    },
+
+    {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
+     "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
+    },
+
+    {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
+     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef libmtp_Device_getsetters[] = {
+    {(char *)"friendly_name", 
+     (getter)libmtp_Device_friendly_name, NULL,
+     (char *)"The friendly name of this device, can be None.",
+     NULL},
+
+    {(char *)"manufacturer_name", 
+     (getter)libmtp_Device_manufacturer_name, NULL,
+     (char *)"The manufacturer name of this device, can be None.",
+     NULL},
+
+    {(char *)"model_name", 
+     (getter)libmtp_Device_model_name, NULL,
+     (char *)"The model name of this device, can be None.",
+     NULL},
+
+    {(char *)"serial_number", 
+     (getter)libmtp_Device_serial_number, NULL,
+     (char *)"The serial number of this device, can be None.",
+     NULL},
+
+    {(char *)"device_version", 
+     (getter)libmtp_Device_device_version, NULL,
+     (char *)"The device version of this device, can be None.",
+     NULL},
+
+    {(char *)"ids", 
+     (getter)libmtp_Device_ids, NULL,
+     (char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
+     NULL},
+
+    {(char *)"storage_info",
+     (getter)libmtp_Device_storage_info, NULL,
+     (char *)"Information about the storage locations on the device. Returns a list of dictionaries where each dictionary corresponds to the LIBMTP_devicestorage_struct.",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject libmtp_DeviceType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "libmtp.Device",            /*tp_name*/
+    sizeof(libmtp_Device),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)libmtp_Device_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Device",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    libmtp_Device_methods,             /* tp_methods */
+    0,             /* tp_members */
+    libmtp_Device_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    (initproc)libmtp_Device_init,      /* tp_init */
+    0,                         /* tp_alloc */
+    0,                 /* tp_new */
+}; // }}}
+
+// }}} End Device object definition
+
+static PyObject *
+libmtp_set_debug_level(PyObject *self, PyObject *args) {
+    int level;
+    if (!PyArg_ParseTuple(args, "i", &level)) return NULL;
+    LIBMTP_Set_Debug(level);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+libmtp_is_mtp_device(PyObject *self, PyObject *args) {
+    int busnum, devnum, vendor_id, prod_id, ans = 0;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiii", &busnum, &devnum, &vendor_id, &prod_id)) return NULL;
+
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == prod_id) {
+            Py_RETURN_TRUE;
+        }
+    }
+
+    /*
+     * LIBMTP_Check_Specific_Device does not seem to work at least on my linux
+     * system. Need to investigate why later. Most devices are in the device
+     * table so this is not terribly important.
+     */
+    /* LIBMTP_Set_Debug(LIBMTP_DEBUG_ALL); */
+    /* printf("Calling check: %d %d\n", busnum, devnum); */
+    Py_BEGIN_ALLOW_THREADS;
+    ans = LIBMTP_Check_Specific_Device(busnum, devnum);
+    Py_END_ALLOW_THREADS;
+
+    if (ans) Py_RETURN_TRUE;
+
+    Py_RETURN_FALSE;
+
+}
+
+static PyMethodDef libmtp_methods[] = {
+    {"set_debug_level", libmtp_set_debug_level, METH_VARARGS,
+        "set_debug_level(level)\n\nSet the debug level bit mask, see LIBMTP_DEBUG_* constants."
+    },
+
+    {"is_mtp_device", libmtp_is_mtp_device, METH_VARARGS,
+        "is_mtp_device(busnum, devnum, vendor_id, prod_id)\n\nReturn True if the device is recognized as an MTP device by its vendor/product ids. If it is not recognized a probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initlibmtp(void) {
+    PyObject *m;
+
+    libmtp_DeviceType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&libmtp_DeviceType) < 0)
+        return;
+    
+    m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
+    if (m == NULL) return;
+
+    LIBMTP_Init();
+    LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);
+
+    Py_INCREF(&libmtp_DeviceType);
+    PyModule_AddObject(m, "Device", (PyObject *)&libmtp_DeviceType);
+
+    PyModule_AddStringMacro(m, LIBMTP_VERSION_STRING);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_NONE);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PTP);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PLST);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_USB);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_DATA);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_ALL);
+}

src/calibre/devices/mtp/unix/upstream/device-flags.h

@@ -0,0 +1,329 @@
+/**
+ * \file device-flags.h
+ * Special device flags to deal with bugs in specific devices.
+ *
+ * Copyright (C) 2005-2007 Richard A. Low <richard@wentnet.com>
+ * Copyright (C) 2005-2012 Linus Walleij <triad@df.lth.se>
+ * Copyright (C) 2006-2007 Marcus Meissner
+ * Copyright (C) 2007 Ted Bullock
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * This file is supposed to be included by both libmtp and libgphoto2.
+ */
+
+/**
+ * These flags are used to indicate if some or other
+ * device need special treatment. These should be possible
+ * to concatenate using logical OR so please use one bit per
+ * feature and lets pray we don't need more than 32 bits...
+ */
+#define DEVICE_FLAG_NONE 0x00000000
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList is broken
+ * in the sense that it won't return properly formatted metadata
+ * for ALL files on the device when you request an object
+ * property list for object 0xFFFFFFFF with parameter 3 likewise
+ * set to 0xFFFFFFFF. Compare to
+ * DEVICE_FLAG_BROKEN_MTPGETOBJECTPROPLIST which only signify
+ * that it's broken when getting metadata for a SINGLE object.
+ * A typical way the implementation may be broken is that it
+ * may not return a proper count of the objects, and sometimes
+ * (like on the ZENs) objects are simply missing from the list
+ * if you use this. Sometimes it has been used incorrectly to
+ * mask bugs in the code (like handling transactions of data
+ * with size given to -1 (0xFFFFFFFFU), in that case please
+ * help us remove it now the code is fixed. Sometimes this is
+ * used because getting all the objects is just too slow and
+ * the USB transaction will time out if you use this command.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL 0x00000001
+/**
+ * This means that under Linux, another kernel module may
+ * be using this device's USB interface, so we need to detach
+ * it if it is. Typically this is on dual-mode devices that
+ * will present both an MTP compliant interface and device
+ * descriptor *and* a USB mass storage interface. If the USB
+ * mass storage interface is in use, other apps (like our
+ * userspace libmtp through libusb access path) cannot get in
+ * and get cosy with it. So we can remove the offending
+ * application. Typically this means you have to run the program
+ * as root as well.
+ */
+#define DEVICE_FLAG_UNLOAD_DRIVER 0x00000002
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList (9805)
+ * is broken in some way, either it doesn't work at all
+ * (as for Android devices) or it won't properly return all
+ * object properties if parameter 3 is set to 0xFFFFFFFFU.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST 0x00000004
+/**
+ * This means the device doesn't send zero packets to indicate
+ * end of transfer when the transfer boundary occurs at a
+ * multiple of 64 bytes (the USB 1.1 endpoint size). Instead,
+ * exactly one extra byte is sent at the end of the transfer
+ * if the size is an integer multiple of USB 1.1 endpoint size
+ * (64 bytes).
+ *
+ * This behaviour is most probably a workaround due to the fact
+ * that the hardware USB slave controller in the device cannot
+ * handle zero writes at all, and the usage of the USB 1.1
+ * endpoint size is due to the fact that the device will "gear
+ * down" on a USB 1.1 hub, and since 64 bytes is a multiple of
+ * 512 bytes, it will work with USB 1.1 and USB 2.0 alike.
+ */
+#define DEVICE_FLAG_NO_ZERO_READS 0x00000008
+/**
+ * This flag means that the device is prone to forgetting the
+ * OGG container file type, so that libmtp must look at the
+ * filename extensions in order to determine that a file is
+ * actually OGG. This is a clear and present firmware bug, and
+ * while firmware bugs should be fixed in firmware, we like
+ * OGG so much that we back it by introducing this flag.
+ * The error has only been seen on iriver devices. Turning this
+ * flag on won't hurt anything, just that the check against
+ * filename extension will be done for files of "unknown" type.
+ * If the player does not even know (reports) that it supports
+ * ogg even though it does, please use the stronger
+ * OGG_IS_UNKNOWN flag, which will forcedly support ogg on
+ * anything with the .ogg filename extension.
+ */
+#define DEVICE_FLAG_IRIVER_OGG_ALZHEIMER 0x00000010
+/**
+ * This flag indicates a limitation in the filenames a device
+ * can accept - they must be 7 bit (all chars <= 127/0x7F).
+ * It was found first on the Philips Shoqbox, and is a deviation
+ * from the PTP standard which mandates that any unicode chars
+ * may be used for filenames. I guess this is caused by a 7bit-only
+ * filesystem being used intrinsically on the device.
+ */
+#define DEVICE_FLAG_ONLY_7BIT_FILENAMES 0x00000020
+/**
+ * This flag indicates that the device will lock up if you
+ * try to get status of endpoints and/or release the interface
+ * when closing the device. This fixes problems with SanDisk
+ * Sansa devices especially. It may be a side-effect of a
+ * Windows behaviour of never releasing interfaces.
+ */
+#define DEVICE_FLAG_NO_RELEASE_INTERFACE 0x00000040
+/**
+ * This flag was introduced with the advent of Creative ZEN
+ * 8GB. The device sometimes return a broken PTP header
+ * like this: < 1502 0000 0200 01d1 02d1 01d2 >
+ * the latter 6 bytes (representing "code" and "transaction ID")
+ * contain junk. This is breaking the PTP/MTP spec but works
+ * on Windows anyway, probably because the Windows implementation
+ * does not check that these bytes are valid. To interoperate
+ * with devices like this, we need this flag to emulate the
+ * Windows bug. Broken headers has also been found in the
+ * Aricent MTP stack.
+ */
+#define DEVICE_FLAG_IGNORE_HEADER_ERRORS 0x00000080
+/**
+ * The Motorola RAZR2 V8 (others?) has broken set object
+ * proplist causing the metadata setting to fail. (The
+ * set object prop to set individual properties work on
+ * this device, but the metadata is plain ignored on
+ * tracks, though e.g. playlist names can be set.)
+ */
+#define DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST 0x00000100
+/**
+ * The Samsung YP-T10 think Ogg files shall be sent with
+ * the "unknown" (PTP_OFC_Undefined) file type, this gives a
+ * side effect that is a combination of the iRiver Ogg Alzheimer
+ * problem (have to recognized Ogg files on file extension)
+ * and a need to report the Ogg support (the device itself does
+ * not properly claim to support it) and need to set filetype
+ * to unknown when storing Ogg files, even though they're not
+ * actually unknown. Later iRivers seem to need this flag since
+ * they do not report to support OGG even though they actually
+ * do. Often the device supports OGG in USB mass storage mode,
+ * then the firmware simply miss to declare metadata support
+ * for OGG properly.
+ */
+#define DEVICE_FLAG_OGG_IS_UNKNOWN 0x00000200
+/**
+ * The Creative Zen is quite unstable in libmtp but seems to
+ * be better with later firmware versions. However, it still
+ * frequently crashes when setting album art dimensions. This
+ * flag disables setting the dimensions (which seems to make
+ * no difference to how the graphic is displayed).
+ */
+#define DEVICE_FLAG_BROKEN_SET_SAMPLE_DIMENSIONS 0x00000400
+/**
+ * Some devices, particularly SanDisk Sansas, need to always
+ * have their "OS Descriptor" probed in order to work correctly.
+ * This flag provides that extra massage.
+ */
+#define DEVICE_FLAG_ALWAYS_PROBE_DESCRIPTOR 0x00000800
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 1.00 is just a simple playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V1 0x00001000
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 2.00 is playlist but allows DNSe sound settings
+ * to be stored, per playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V2 0x00002000
+/**
+ * The Sansa E250 is know to have this problem which is actually
+ * that the device claims that property PTP_OPC_DateModified
+ * is read/write but will still fail to update it. It can only
+ * be set properly the first time a file is sent.
+ */
+#define DEVICE_FLAG_CANNOT_HANDLE_DATEMODIFIED 0x00004000
+/**
+ * This avoids use of the send object proplist which
+ * is used when creating new objects (not just updating)
+ * The DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST is related
+ * but only concerns the case where the object proplist
+ * is sent in to update an existing object. The Toshiba
+ * Gigabeat MEU202 for example has this problem.
+ */
+#define DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST 0x00008000
+/**
+ * Devices that cannot support reading out battery
+ * level.
+ */
+#define DEVICE_FLAG_BROKEN_BATTERY_LEVEL 0x00010000
+
+/**
+ * Devices that send "ObjectDeleted" events after deletion
+ * of images. (libgphoto2)
+ */
+#define DEVICE_FLAG_DELETE_SENDS_EVENT	0x00020000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE		0x00040000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE_PREVIEW	0x00080000
+
+/**
+ * Nikon broken capture support without proper ObjectAdded events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NIKON_BROKEN_CAPTURE	0x00100000
+
+/**
+ * Broken capture support where cameras do not send CaptureComplete events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NO_CAPTURE_COMPLETE		0x00400000
+
+/**
+ * Direct PTP match required.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_MATCH_PTP_INTERFACE		0x00800000
+/**
+ * This flag is like DEVICE_FLAG_OGG_IS_UNKNOWN but for FLAC
+ * files instead. Using the unknown filetype for FLAC files.
+ */
+#define DEVICE_FLAG_FLAC_IS_UNKNOWN		0x01000000
+/**
+ * Device needs unique filenames, no two files can be
+ * named the same string.
+ */
+#define DEVICE_FLAG_UNIQUE_FILENAMES		0x02000000
+/**
+ * This flag performs some random magic on the BlackBerry
+ * device to switch from USB mass storage to MTP mode we think.
+ */
+#define DEVICE_FLAG_SWITCH_MODE_BLACKBERRY	0x04000000
+/**
+ * This flag indicates that the device need an extra long
+ * timeout on some operations.
+ */
+#define DEVICE_FLAG_LONG_TIMEOUT		0x08000000
+/**
+ * This flag indicates that the device need an explicit
+ * USB reset after each connection. Some devices don't
+ * like this, so it's not done by default.
+ */
+#define DEVICE_FLAG_FORCE_RESET_ON_CLOSE	0x10000000
+/**
+ * Early Creative Zen (etc) models actually only support
+ * command 9805 (Get object property list) and will hang
+ * if you try to get individual properties of an object.
+ */
+#define DEVICE_FLAG_BROKEN_GET_OBJECT_PROPVAL	0x20000000
+/**
+ * It seems that some devices return an bad data when
+ * using the GetObjectInfo operation. So in these cases
+ * we prefer to override the PTP-compatible object infos
+ * with the MTP property list.
+ *
+ * For example Some Samsung Galaxy S devices contain an MTP
+ * stack that present the ObjectInfo in 64 bit instead of
+ * 32 bit.
+ */
+#define DEVICE_FLAG_PROPLIST_OVERRIDES_OI	0x40000000
+
+/**
+ * All these bug flags need to be set on SONY NWZ Walkman
+ * players, and will be autodetected on unknown devices
+ * by detecting the vendor extension descriptor "sony.net"
+ */
+#define DEVICE_FLAGS_SONY_NWZ_BUGS \
+  (DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_UNIQUE_FILENAMES | \
+   DEVICE_FLAG_FORCE_RESET_ON_CLOSE )
+/**
+ * All these bug flags need to be set on Android devices,
+ * they claim to support MTP operations they actually
+ * cannot handle, especially 9805 (Get object property list).
+ * These are auto-assigned to devices reporting
+ * "android.com" in their device extension descriptor.
+ */
+#define DEVICE_FLAGS_ANDROID_BUGS \
+  (DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_LONG_TIMEOUT )
+/**
+ * All these bug flags appear on a number of SonyEricsson
+ * devices including Android devices not using the stock
+ * Android 4.0+ (Ice Cream Sandwich) MTP stack. It is highly
+ * supected that these bugs comes from an MTP implementation
+ * from Aricent, so it is called the Aricent bug flags as a
+ * shorthand. Especially the header errors that need to be
+ * ignored is typical for this stack.
+ *
+ * After some guesswork we auto-assign these bug flags to
+ * devices that present the "microsoft.com/WPDNA", and
+ * "sonyericsson.com/SE" but NOT the "android.com"
+ * descriptor.
+ */
+#define DEVICE_FLAGS_ARICENT_BUGS \
+  (DEVICE_FLAG_IGNORE_HEADER_ERRORS | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST )

src/calibre/devices/mtp/unix/upstream/update.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+MP = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/music-players.h;hb=HEAD'
+DF = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/device-flags.h;hb=HEAD'
+
+import urllib, os, shutil
+
+base = os.path.dirname(os.path.abspath(__file__))
+
+for url, fname in [(MP, 'music-players.h'), (DF, 'device-flags.h')]:
+    with open(os.path.join(base, fname), 'wb') as f:
+        shutil.copyfileobj(urllib.urlopen(url), f)
+

src/calibre/devices/scanner.py

@@ -7,6 +7,7 @@ manner.
 
 import sys, os, re
 from threading import RLock
+from collections import namedtuple
 
 from calibre import prints, as_unicode
 from calibre.constants import iswindows, isosx, plugins, islinux, isfreebsd
@@ -107,6 +108,15 @@ class WinPNPScanner(object):
 
 win_pnp_drives = WinPNPScanner()
 
+_USBDevice = namedtuple('USBDevice',
+    'vendor_id product_id bcd manufacturer product serial')
+
+class USBDevice(_USBDevice):
+
+    def __init__(self, *args, **kwargs):
+        _USBDevice.__init__(self, *args, **kwargs)
+        self.busnum = self.devnum = -1
+
 class LinuxScanner(object):
 
     SYSFS_PATH = os.environ.get('SYSFS_PATH', '/sys')
@@ -122,6 +132,10 @@ class LinuxScanner(object):
         if not self.ok:
             raise RuntimeError('DeviceScanner requires the /sys filesystem to work.')
 
+        def read(f):
+            with open(f, 'rb') as s:
+                return s.read().strip()
+
         for x in os.listdir(self.base):
             base = os.path.join(self.base, x)
             ven = os.path.join(base, 'idVendor')
@@ -132,31 +146,46 @@ class LinuxScanner(object):
             prod_string = os.path.join(base, 'product')
             dev = []
             try:
-                dev.append(int('0x'+open(ven).read().strip(), 16))
+                # Ignore USB HUBs
+                if read(os.path.join(base, 'bDeviceClass')) == b'09':
+                    continue
             except:
                 continue
             try:
-                dev.append(int('0x'+open(prod).read().strip(), 16))
+                dev.append(int(b'0x'+read(ven), 16))
             except:
                 continue
             try:
-                dev.append(int('0x'+open(bcd).read().strip(), 16))
+                dev.append(int(b'0x'+read(prod), 16))
             except:
                 continue
             try:
-                dev.append(open(man).read().strip())
+                dev.append(int(b'0x'+read(bcd), 16))
             except:
-                dev.append('')
+                continue
             try:
-                dev.append(open(prod_string).read().strip())
+                dev.append(read(man))
             except:
-                dev.append('')
+                dev.append(b'')
             try:
-                dev.append(open(serial).read().strip())
+                dev.append(read(prod_string))
             except:
-                dev.append('')
+                dev.append(b'')
+            try:
+                dev.append(read(serial))
+            except:
+                dev.append(b'')
 
-            ans.add(tuple(dev))
+            dev = USBDevice(*dev)
+            try:
+                dev.busnum = int(read(os.path.join(base, 'busnum')))
+            except:
+                pass
+            try:
+                dev.devnum = int(read(os.path.join(base, 'devnum')))
+            except:
+                pass
+            ans.add(dev)
         return ans
 
 class FreeBSDScanner(object):

src/calibre/devices/smart_device_app/driver.py

@@ -17,7 +17,7 @@ from calibre.constants import numeric_version, DEBUG
 from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
                                     InitialConnectionError)
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.usbms.books import Book, BookList
+from calibre.devices.usbms.books import Book, CollectionsBookList
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.usbms.driver import USBMS
 from calibre.ebooks import BOOK_EXTENSIONS
@@ -107,6 +107,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     }
     reverse_opcodes = dict([(v, k) for k,v in opcodes.iteritems()])
 
+    ALL_BY_TITLE  = _('All by title')
+    ALL_BY_AUTHOR = _('All by author')
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
         _('Enable connections at startup') + ':::<p>' +
@@ -122,6 +124,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             _('Enter the port number the driver is to use if the "fixed port" box is checked') + '</p>',
         _('Print extra debug information') + ':::<p>' +
             _('Check this box if requested when reporting problems') + '</p>',
+        '',
+        _('Comma separated list of metadata fields '
+            'to turn into collections on the device. Possibilities include: ')+\
+                    'series, tags, authors' +\
+            _('. Two special collections are available: %(abt)s:%(abtv)s and %(aba)s:%(abav)s. Add  '
+            'these values to the list to enable them. The collections will be '
+            'given the name provided after the ":" character.')%dict(
+                            abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR)
         ]
     EXTRA_CUSTOMIZATION_DEFAULT = [
                 False,
@@ -130,12 +140,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 '',
                 False, '9090',
                 False,
+                '',
+                ''
     ]
     OPT_AUTOSTART               = 0
     OPT_PASSWORD                = 2
     OPT_USE_PORT                = 4
     OPT_PORT_NUMBER             = 5
     OPT_EXTRA_DEBUG             = 6
+    OPT_COLLECTIONS             = 8
 
     def __init__(self, path):
         self.sync_lock = threading.RLock()
@@ -586,7 +599,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     # the device.
                     raise OpenFailed('')
             try:
-                peer = self.device_socket.getpeername()
+                peer = self.device_socket.getpeername()[0]
                 self.connection_attempts[peer] = 0
             except:
                 pass
@@ -659,9 +672,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     def books(self, oncard=None, end_session=True):
         self._debug(oncard)
         if oncard is not None:
-            return BookList(None, None, None)
+            return CollectionsBookList(None, None, None)
         opcode, result = self._call_client('GET_BOOK_COUNT', {})
-        bl = BookList(None, self.PREFIX, self.settings)
+        bl = CollectionsBookList(None, self.PREFIX, self.settings)
         if opcode == 'OK':
             count = result['count']
             for i in range(0, count):
@@ -680,11 +693,23 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def sync_booklists(self, booklists, end_session=True):
-        self._debug()
+        colattrs = [x.strip() for x in
+                self.settings().extra_customization[self.OPT_COLLECTIONS].split(',')]
+        self._debug('collection attributes', colattrs)
+        coldict = {}
+        if colattrs:
+            collections = booklists[0].get_collections(colattrs)
+            for k,v in collections.iteritems():
+                lpaths = []
+                for book in v:
+                    lpaths.append(book.lpath)
+                coldict[k] = lpaths
+
         # If we ever do device_db plugboards, this is where it will go. We will
         # probably need to send two booklists, one with calibre's data that is
         # given back by "books", and one that has been plugboarded.
-        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]) } )
+        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]),
+                                              'collections': coldict} )
         for i,book in enumerate(booklists[0]):
             if not self._metadata_already_on_device(book):
                 self._set_known_metadata(book)

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -73,7 +73,10 @@ class HTMLTOCAdder(object):
                 if (hasattr(item.data, 'xpath') and
                     XPath('//h:a[@href]')(item.data)):
                     if oeb.spine.index(item) < 0:
-                        oeb.spine.add(item, linear=False)
+                        if self.position == 'end':
+                            oeb.spine.add(item, linear=False)
+                        else:
+                            oeb.spine.insert(0, item, linear=True)
                     return
                 elif has_toc:
                     oeb.guide.remove('toc')

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -414,7 +414,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
                      and fm[f].get('search_terms', None)
                      and f not in ['formats', 'ondevice']) or
                     (fm[f]['datatype'] in ['int', 'float', 'bool', 'datetime'] and
-                     f not in ['id'])):
+                     f not in ['id', 'timestamp'])):
                 self.all_fields.append(f)
                 self.writable_fields.append(f)
             if fm[f]['datatype'] == 'composite':

src/calibre/gui2/tag_browser/model.py

@@ -425,7 +425,8 @@ class TagsModel(QAbstractItemModel): # {{{
             last_idx = -collapse
             category_is_hierarchical = not (
                 key in ['authors', 'publisher', 'news', 'formats', 'rating'] or
-                key not in self.db.prefs.get('categories_using_hierarchy', []))
+                key not in self.db.prefs.get('categories_using_hierarchy', []) or
+                config['sort_tags_by'] != 'name')
 
             for idx,tag in enumerate(data[key]):
                 components = None

src/calibre/gui2/viewer/documentview.py

@@ -1036,14 +1036,14 @@ class DocumentView(QWebView): # {{{
         if not self.handle_key_press(event):
             return QWebView.keyPressEvent(self, event)
 
-    def paged_col_scroll(self, forward=True):
+    def paged_col_scroll(self, forward=True, scroll_past_end=True):
         dir = 'next' if forward else 'previous'
         loc = self.document.javascript(
                 'paged_display.%s_col_location()'%dir, typ='int')
         if loc > -1:
             self.document.scroll_to(x=loc, y=0)
             self.manager.scrolled(self.document.scroll_fraction)
-        else:
+        elif scroll_past_end:
             (self.manager.next_document() if forward else
                     self.manager.previous_document())
 
@@ -1059,7 +1059,8 @@ class DocumentView(QWebView): # {{{
                 self.is_auto_repeat_event = False
         elif key == 'Down':
             if self.document.in_paged_mode:
-                self.paged_col_scroll()
+                self.paged_col_scroll(scroll_past_end=not
+                        self.document.line_scrolling_stops_on_pagebreaks)
             else:
                 if (not self.document.line_scrolling_stops_on_pagebreaks and
                         self.document.at_bottom):
@@ -1068,7 +1069,8 @@ class DocumentView(QWebView): # {{{
                     self.scroll_by(y=15)
         elif key == 'Up':
             if self.document.in_paged_mode:
-                self.paged_col_scroll(forward=False)
+                self.paged_col_scroll(forward=False, scroll_past_end=not
+                        self.document.line_scrolling_stops_on_pagebreaks)
             else:
                 if (not self.document.line_scrolling_stops_on_pagebreaks and
                         self.document.at_top):

src/calibre/library/database2.py

@@ -31,7 +31,8 @@ from calibre.ptempfile import (PersistentTemporaryFile,
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
-from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
+from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
+        parse_only_date)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
@@ -2479,6 +2480,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_pubdate(self, id, dt, notify=True, commit=True):
         if dt:
+            if isinstance(dt, basestring):
+                dt = parse_only_date(dt)
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
             self.dirtied([id], commit=False)