0.8.63+

manual/Makefile

@@ -60,7 +60,7 @@ htmlhelp:
 
 latex:
 	mkdir -p .build/latex .build/doctrees
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) .build/latex
+	$(SPHINXBUILD) -b mylatex $(ALLSPHINXOPTS) .build/latex
 	@echo
 	@echo "Build finished; the LaTeX files are in .build/latex."
 	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \

manual/conf.py

@@ -14,10 +14,10 @@
 import sys, os
 
 # If your extensions are in another directory, add it here.
-sys.path.append(os.path.abspath('../src'))
 sys.path.append(os.path.abspath('.'))
-__appname__ = os.environ.get('__appname__', 'calibre')
+import init_calibre
-__version__ = os.environ.get('__version__', '0.0.0')
+init_calibre
+from calibre.constants import __appname__, __version__
 import custom
 custom
 # General configuration
@@ -154,7 +154,8 @@ latex_font_size = '10pt'
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
-#latex_documents = []
+latex_documents = [('index', 'calibre.tex', 'calibre User Manual',
+    'Kovid Goyal', 'manual', False)]
 
 # Additional stuff for the LaTeX preamble.
 #latex_preamble = ''
@@ -164,3 +165,11 @@ latex_font_size = '10pt'
 
 # If false, no module index is generated.
 #latex_use_modindex = True
+
+latex_logo = 'resources/logo.png'
+latex_show_pagerefs = True
+latex_show_urls = 'footnote'
+latex_elements = {
+'papersize':'letterpaper',
+'fontenc':r'\usepackage[T2A,T1]{fontenc}'
+}

manual/custom.py

@@ -14,6 +14,7 @@ from sphinx.util.console import bold
 sys.path.append(os.path.abspath('../../../'))
 from calibre.linux import entry_points
 from epub import EPUBHelpBuilder
+from latex import LaTeXHelpBuilder
 
 def substitute(app, doctree):
     pass
@@ -251,6 +252,7 @@ def template_docs(app):
 def setup(app):
     app.add_config_value('kovid_epub_cover', None, False)
     app.add_builder(EPUBHelpBuilder)
+    app.add_builder(LaTeXHelpBuilder)
     app.connect('doctree-read', substitute)
     app.connect('builder-inited', generate_docs)
     app.connect('build-finished', finished)

manual/index.rst

@@ -17,7 +17,7 @@ To get started with more advanced usage, you should read about the :ref:`Graphic
 
 .. only:: online
     
-    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_ and `AZW3 (Kindle Fire) format <calibre.azw3>`_.
+    **An ebook version of this user manual is available in** `EPUB format <calibre.epub>`_,  `AZW3 (Kindle Fire) format <calibre.azw3>`_ and `PDF format <calibre.pdf>`_.
     
 Sections
 ------------

manual/latex.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+
+from sphinx.builders.latex import LaTeXBuilder
+
+class LaTeXHelpBuilder(LaTeXBuilder):
+    name = 'mylatex'
+
+    def finish(self):
+        LaTeXBuilder.finish(self)
+        self.info('Fixing Cyrillic characters...')
+        tex = os.path.join(self.outdir, 'calibre.tex')
+        with open(tex, 'r+b') as f:
+            raw = f.read().replace(b'Михаил Горбачёв',
+                br'{\fontencoding{T2A}\selectfont Михаил Горбачёв}')
+            f.seek(0)
+            f.write(raw)

recipes/ekundelek_pl.recipe

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+__license__ = 'GPL v3'
+__copyright__ = u'2012, Artur Stachecki <artur.stachecki@gmail.com>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class swiatczytnikow(BasicNewsRecipe):
+    title = u'eKundelek'
+    description = u'Najsympatyczniejszy blog o e-czytnikach Kindle'
+    language = 'pl'
+    __author__ = u'Artur Stachecki'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    remove_tags = [dict(name = 'div', attrs = {'class' : 'feedflare'})]
+
+    feeds = [(u'Wpisy', u'http://feeds.feedburner.com/Ekundelekpl?format=xml')]

recipes/metro_uk.recipe

@@ -1,31 +1,42 @@
 from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1306097511(BasicNewsRecipe):
     title          = u'Metro UK'
-    description = 'News as provide by The Metro -UK'
+    description = 'Author Dave Asbury : News as provide by The Metro -UK'
     #timefmt = ''
     __author__ = 'Dave Asbury'
-    #last update 9/6/12
+    #last update 4/8/12
     cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg'
-    #no_stylesheets = True
+    no_stylesheets = True
     oldest_article = 1
-    max_articles_per_feed = 10
+    max_articles_per_feed = 12
     remove_empty_feeds = True
     remove_javascript     = True
-    auto_cleanup = True
+    #auto_cleanup = True
     encoding = 'UTF-8'
-
+    cover_url ='http://profile.ak.fbcdn.net/hprofile-ak-snc4/157897_117118184990145_840702264_n.jpg'
     language = 'en_GB'
     masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:1.6em;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:1.2em;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:1.0em;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:1.0em;}
+		'''
     keep_only_tags = [
-
+		#dict(name='h1'),
+		#dict(name='h2'),
+		#dict(name='div', attrs={'class' : ['row','article','img-cnt figure','clrd']})
+	#dict(name='h3'),
+	#dict(attrs={'class' :  'BText'}),
 		]
     remove_tags    = [
-
+                                dict(name='span',attrs={'class' : 'share'}),
+		dict(name='li'),
+		dict(attrs={'class' : ['twitter-share-button','header-forms','hdr-lnks','close','art-rgt','fd-gr1-b clrd google-article','news m12 clrd clr-b p5t shareBtm','item-ds csl-3-img news','c-1of3 c-last','c-1of1','pd','item-ds csl-3-img sport']}),
+           		dict(attrs={'id' : ['','sky-left','sky-right','ftr-nav','and-ftr','notificationList','logo','miniLogo','comments-news','metro_extras']})
                                ]
-
+    remove_tags_before = dict(name='h1')
+    #remove_tags_after = dict(attrs={'id':['topic-buttons']})
 
     feeds          = [
         (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
-    extra_css  = '''
-	body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
-                	 '''

recipes/sueddeutsche_mobil.recipe

@@ -0,0 +1,117 @@
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+__license__   = 'GPL v3'
+__copyright__ = '2012, Andreas Zeiser <andreas.zeiser@web.de>'
+'''
+szmobil.sueddeutsche.de/
+'''
+
+from calibre import strftime
+from calibre.web.feeds.recipes import BasicNewsRecipe
+import re
+
+class SZmobil(BasicNewsRecipe):
+    title       = u'Süddeutsche Zeitung mobil'
+    __author__  = u'Andreas Zeiser'
+    description = u'Nachrichten aus Deutschland. Zugriff auf kostenpflichtiges Abo SZ mobil.'
+    publisher              = u'Sueddeutsche Zeitung'
+    language               = u'de'
+    publication_type       = u'newspaper'
+    category               = u'news, politics, Germany'
+
+    no_stylesheets         = True
+    oldest_article         = 2
+    encoding               = 'iso-8859-1'
+    needs_subscription     = True
+    remove_empty_feeds     = True
+    delay                  = 1
+    cover_source           = 'http://www.sueddeutsche.de/verlag'
+
+    timefmt = ' [%a, %d %b, %Y]'
+
+    root_url ='http://szmobil.sueddeutsche.de/'
+    keep_only_tags = [dict(name='div', attrs={'class':'article'})]
+
+    def get_cover_url(self):
+        src = self.index_to_soup(self.cover_source)
+        image_url = src.find(attrs={'class':'preview-image'})
+        return image_url.div.img['src']
+
+    def get_browser(self):
+        browser = BasicNewsRecipe.get_browser(self)
+
+        # Login via fetching of Streiflicht -> Fill out login request
+        url = self.root_url + 'show.php?id=streif'
+        browser.open(url)
+
+        browser.select_form(nr=0) # to select the first form
+        browser['username'] = self.username
+        browser['password'] = self.password
+        browser.submit()
+
+        return browser
+
+	def parse_index(self):
+        # find all sections
+        src = self.index_to_soup('http://szmobil.sueddeutsche.de')
+        feeds = []
+        for itt in src.findAll('a',href=True):
+            if itt['href'].startswith('show.php?section'):
+                feeds.append( (itt.string[0:-2],itt['href']) )
+
+        all_articles = []
+        for feed in feeds:
+            feed_url = self.root_url + feed[1]
+            feed_title = feed[0]
+
+            self.report_progress(0, ('Fetching feed')+' %s...'%(feed_title if feed_title else feed_url))
+
+            src = self.index_to_soup(feed_url)
+            articles = []
+            shorttitles = dict()
+            for itt in src.findAll('a', href=True):
+                if itt['href'].startswith('show.php?id='):
+                    article_url = itt['href']
+                    article_id = int(re.search("id=(\d*)&etag=", itt['href']).group(1))
+
+                    # first check if link is a special article in section "Meinungsseite"
+                    if itt.find('strong')!= None:
+                        article_name = itt.strong.string
+                        article_shorttitle = itt.contents[1]
+
+                        articles.append( (article_name, article_url, article_id) )
+                        shorttitles[article_id] = article_shorttitle
+                        continue
+
+
+                    # candidate for a general article
+                    if itt.string == None:
+                        article_name = ''
+                    else:
+                        article_name = itt.string
+
+                    if (article_name[0:10] == "&nbsp;mehr"):
+                        # just another link ("mehr") to an article
+                        continue
+
+                    if itt.has_key('id'):
+                        shorttitles[article_id] = article_name
+                    else:
+                        articles.append( (article_name, article_url, article_id) )
+
+            feed_articles = []
+            for article_name, article_url, article_id in articles:
+                url = self.root_url + article_url
+                title = article_name
+                pubdate = strftime('%a, %d %b')
+                description = ''
+                if shorttitles.has_key(article_id):
+                    description = shorttitles[article_id]
+                # we do not want the flag ("Impressum")
+                if "HERAUSGEGEBEN VOM" in description:
+                    continue
+                d = dict(title=title, url=url, date=pubdate, description=description, content='')
+                feed_articles.append(d)
+            all_articles.append( (feed_title, feed_articles) )
+
+        return all_articles
+

resources/catalog/stylesheet.css

@@ -151,14 +151,23 @@ p.title  {
 	font-size:xx-large;
 	}
 
-p.wishlist_item, p.unread_book, p.read_book {
+p.wishlist_item, p.unread_book, p.read_book, p.line_item {
-	text-align:left;
+    font-family:monospace;
 	margin-top:0px;
 	margin-bottom:0px;
 	margin-left:2em;
+	text-align:left;
 	text-indent:-2em;
 	}
 
+span.prefix {}
+span.entry {
+    font-family: serif;
+    }
+
+/*
+*   Book Descriptions
+*/
 td.publisher, td.date {
 	font-weight:bold;
 	text-align:center;

setup/extensions.py

@@ -174,6 +174,20 @@ if isosx:
                 ldflags=['-framework', 'IOKit'])
             )
 
+if islinux:
+    extensions.append(Extension('libmtp',
+        [
+        'calibre/devices/mtp/unix/devices.c',
+        'calibre/devices/mtp/unix/libmtp.c'
+        ],
+        headers=[
+        'calibre/devices/mtp/unix/devices.h',
+        'calibre/devices/mtp/unix/upstream/music-players.h',
+        'calibre/devices/mtp/unix/upstream/device-flags.h',
+        ],
+        libraries=['mtp']
+    ))
+
 if isunix:
     cc = os.environ.get('CC', 'gcc')
     cxx = os.environ.get('CXX', 'g++')

setup/publish.py

@@ -80,8 +80,17 @@ class Manual(Command):
                                    '-d', '.build/doctrees', '.', '.build/html'])
             subprocess.check_call(['sphinx-build', '-b', 'myepub', '-d',
                                    '.build/doctrees', '.', '.build/epub'])
+            subprocess.check_call(['sphinx-build', '-b', 'mylatex', '-d',
+                                   '.build/doctrees', '.', '.build/latex'])
+            pwd = os.getcwdu()
+            os.chdir('.build/latex')
+            subprocess.check_call(['make', 'all-pdf'], stdout=open(os.devnull,
+                'wb'))
+            os.chdir(pwd)
             epub_dest = self.j('.build', 'html', 'calibre.epub')
+            pdf_dest = self.j('.build', 'html', 'calibre.pdf')
             shutil.copyfile(self.j('.build', 'epub', 'calibre.epub'), epub_dest)
+            shutil.copyfile(self.j('.build', 'latex', 'calibre.pdf'), pdf_dest)
             subprocess.check_call(['ebook-convert', epub_dest,
                 epub_dest.rpartition('.')[0] + '.azw3',
                 '--page-breaks-before=/', '--disable-font-rescaling',

src/calibre/constants.py

@@ -93,6 +93,8 @@ class Plugins(collections.Mapping):
             plugins.append('winutil')
         if isosx:
             plugins.append('usbobserver')
+        if islinux:
+            plugins.append('libmtp')
         self.plugins = frozenset(plugins)
 
     def load_plugin(self, name):

src/calibre/customize/profiles.py

@@ -251,10 +251,8 @@ class OutputProfile(Plugin):
     periodical_date_in_title = True
 
     #: Characters used in jackets and catalogs
-    missing_char = u'x'
     ratings_char = u'*'
     empty_ratings_char = u' '
-    read_char = u'+'
 
     #: Unsupported unicode characters to be replaced during preprocessing
     unsupported_unicode_chars = []
@@ -292,10 +290,8 @@ class iPadOutput(OutputProfile):
         }
     ]
 
-    missing_char = u'\u2715\u200a'      # stylized 'x' plus hair space
     ratings_char = u'\u2605'            # filled star
     empty_ratings_char = u'\u2606'      # hollow star
-    read_char = u'\u2713'               # check mark
 
     touchscreen = True
     # touchscreen_news_css {{{
@@ -626,10 +622,8 @@ class KindleOutput(OutputProfile):
     supports_mobi_indexing = True
     periodical_date_in_title = False
 
-    missing_char = u'x\u2009'
     empty_ratings_char = u'\u2606'
     ratings_char = u'\u2605'
-    read_char = u'\u2713'
 
     mobi_ems_per_blockquote = 2.0
 
@@ -651,10 +645,8 @@ class KindleDXOutput(OutputProfile):
     #comic_screen_size         = (741, 1022)
     supports_mobi_indexing = True
     periodical_date_in_title = False
-    missing_char = u'x\u2009'
     empty_ratings_char = u'\u2606'
     ratings_char = u'\u2605'
-    read_char = u'\u2713'
     mobi_ems_per_blockquote = 2.0
 
     @classmethod

src/calibre/devices/android/driver.py

@@ -43,6 +43,7 @@ class ANDROID(USBMS):
                        0xccf  : HTC_BCDS,
                        0xcd6  : HTC_BCDS,
                        0xce5  : HTC_BCDS,
+                       0xcec  : HTC_BCDS,
                        0x2910 : HTC_BCDS,
                        0xff9  : HTC_BCDS,
             },

src/calibre/devices/interface.py

@@ -297,7 +297,7 @@ class DevicePlugin(Plugin):
 
         :return: (device name, device version, software version on device, mime type)
                  The tuple can optionally have a fifth element, which is a
-                 drive information diction. See usbms.driver for an example.
+                 drive information dictionary. See usbms.driver for an example.
 
         """
         raise NotImplementedError()
@@ -607,7 +607,7 @@ class BookList(list):
         pass
 
     def supports_collections(self):
-        ''' Return True if the the device supports collections for this book list. '''
+        ''' Return True if the device supports collections for this book list. '''
         raise NotImplementedError()
 
     def add_book(self, book, replace_metadata):

src/calibre/devices/mtp/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/devices/mtp/base.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.devices.interface import DevicePlugin
+
+class MTPDeviceBase(DevicePlugin):
+    name = 'SmartDevice App Interface'
+    gui_name = _('MTP Device')
+    icon = I('devices/galaxy_s3.png')
+    description = _('Communicate with MTP devices')
+    author = 'Kovid Goyal'
+    version = (1, 0, 0)
+
+    # Invalid USB vendor information so the scanner will never match
+    VENDOR_ID                   = [0xffff]
+    PRODUCT_ID                  = [0xffff]
+    BCD                         = [0xffff]
+
+    THUMBNAIL_HEIGHT = 128
+    CAN_SET_METADATA = []
+
+    BACKLOADING_ERROR_MESSAGE = None
+
+    def __init__(self, *args, **kwargs):
+        DevicePlugin.__init__(self, *args, **kwargs)
+        self.progress_reporter = None
+
+    def reset(self, key='-1', log_packets=False, report_progress=None,
+            detected_device=None):
+        pass
+
+    def set_progress_reporter(self, report_progress):
+        self.progress_reporter = report_progress
+

src/calibre/devices/mtp/unix/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+'''
+libmtp based drivers for MTP devices on Unix like platforms.
+'''
+

src/calibre/devices/mtp/unix/detect.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.constants import plugins
+
+class MTPDetect(object):
+
+    def __init__(self):
+        p = plugins['libmtp']
+        self.libmtp = p[0]
+        if self.libmtp is None:
+            print ('Failed to load libmtp, MTP device detection disabled')
+            print (p[1])
+        self.cache = {}
+
+    def __call__(self, devices):
+        '''
+        Given a list of devices as returned by LinuxScanner, return the set of
+        devices that are likely to be MTP devices. This class maintains a cache
+        to minimize USB polling. Note that detection is partially based on a
+        list of known vendor and product ids. This is because polling some
+        older devices causes problems. Therefore, if this method identifies a
+        device as MTP, it is not actually guaranteed that it will be a working
+        MTP device.
+        '''
+        # First drop devices that have been disconnected from the cache
+        connected_devices = {(d.busnum, d.devnum, d.vendor_id, d.product_id,
+            d.bcd, d.serial) for d in devices}
+        for d in tuple(self.cache.iterkeys()):
+            if d not in connected_devices:
+                del self.cache[d]
+
+        # Since is_mtp_device() can cause USB traffic by probing the device, we
+        # cache its result
+        mtp_devices = set()
+        if self.libmtp is None:
+            return mtp_devices
+
+        for d in devices:
+            ans = self.cache.get((d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial), None)
+            if ans is None:
+                ans = self.libmtp.is_mtp_device(d.busnum, d.devnum,
+                        d.vendor_id, d.product_id)
+                self.cache[(d.busnum, d.devnum, d.vendor_id, d.product_id,
+                    d.bcd, d.serial)] = ans
+            if ans:
+                mtp_devices.add(d)
+        return mtp_devices
+
+    def create_device(self, connected_device):
+        d = connected_device
+        return self.libmtp.Device(d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.manufacturer, d.product, d.serial)
+
+if __name__ == '__main__':
+    from calibre.devices.scanner import linux_scanner
+    mtp_detect = MTPDetect()
+    devs = mtp_detect(linux_scanner())
+    print ('Found %d MTP devices:'%len(devs))
+    for dev in devs:
+        print (dev, 'at busnum=%d and devnum=%d'%(dev.busnum, dev.devnum))
+        print()
+
+

src/calibre/devices/mtp/unix/devices.c

@@ -0,0 +1,16 @@
+/*
+ * devices.c
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+
+#include "upstream/device-flags.h"
+#include "devices.h"
+
+const calibre_device_entry_t calibre_mtp_device_table[] = {
+#include "upstream/music-players.h"
+
+  , { NULL, 0xffff, NULL, 0xffff, DEVICE_FLAG_NONE }
+};
+

src/calibre/devices/mtp/unix/devices.h

@@ -0,0 +1,22 @@
+#pragma once
+/*
+ * devices.h
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the MIT license.
+ */
+#include <stdint.h>
+#include <stddef.h>
+
+struct calibre_device_entry_struct {
+  char *vendor; /**< The vendor of this device */
+  uint16_t vendor_id; /**< Vendor ID for this device */
+  char *product; /**< The product name of this device */
+  uint16_t product_id; /**< Product ID for this device */
+  uint32_t device_flags; /**< Bugs, device specifics etc */
+};
+
+typedef struct calibre_device_entry_struct calibre_device_entry_t;
+
+extern const calibre_device_entry_t calibre_mtp_device_table[];
+

src/calibre/devices/mtp/unix/driver.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time, operator
+from threading import RLock
+from functools import wraps
+
+from calibre.devices.errors import OpenFailed
+from calibre.devices.mtp.base import MTPDeviceBase
+from calibre.devices.mtp.unix.detect import MTPDetect
+
+def synchronous(func):
+    @wraps(func)
+    def synchronizer(self, *args, **kwargs):
+        with self.lock:
+            return func(self, *args, **kwargs)
+    return synchronizer
+
+class MTP_DEVICE(MTPDeviceBase):
+
+    supported_platforms = ['linux']
+
+    def __init__(self, *args, **kwargs):
+        MTPDeviceBase.__init__(self, *args, **kwargs)
+        self.detect = MTPDetect()
+        self.dev = None
+        self.lock = RLock()
+        self.blacklisted_devices = set()
+
+    def report_progress(self, sent, total):
+        try:
+            p = int(sent/total * 100)
+        except ZeroDivisionError:
+            p = 100
+        if self.progress_reporter is not None:
+            self.progress_reporter(p)
+
+    @synchronous
+    def get_gui_name(self):
+        if self.dev is None or not self.dev.friendly_name: return self.name
+        return self.dev.friendly_name
+
+    @synchronous
+    def is_usb_connected(self, devices_on_system, debug=False,
+            only_presence=False):
+
+        # First remove blacklisted devices.
+        devs = []
+        for d in devices_on_system:
+            if (d.busnum, d.devnum, d.vendor_id,
+                d.product_id, d.bcd, d.serial) not in self.blacklisted_devices:
+                devs.append(d)
+
+        devs = self.detect(devs)
+        if self.dev is not None:
+            # Check if the currently opened device is still connected
+            ids = self.dev.ids
+            found = False
+            for d in devs:
+                if ( (d.busnum, d.devnum, d.vendor_id, d.product_id, d.serial)
+                        == ids ):
+                    found = True
+                    break
+            return found
+        # Check if any MTP capable device is present
+        return len(devs) > 0
+
+    @synchronous
+    def post_yank_cleanup(self):
+        self.dev = None
+
+    @synchronous
+    def shutdown(self):
+        self.dev = None
+
+    @synchronous
+    def open(self, connected_device, library_uuid):
+        def blacklist_device():
+            d = connected_device
+            self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id,
+                    d.product_id, d.bcd, d.serial))
+        try:
+            self.dev = self.detect.create_device(connected_device)
+        except ValueError:
+            # Give the device some time to settle
+            time.sleep(2)
+            try:
+                self.dev = self.detect.create_device(connected_device)
+            except ValueError:
+                # Black list this device so that it is ignored for the
+                # remainder of this session.
+                blacklist_device()
+                raise OpenFailed('%s is not a MTP device'%(connected_device,))
+        except TypeError:
+            blacklist_device()
+            raise OpenFailed('')
+
+        storage = sorted(self.dev.storage_info, key=operator.itemgetter('id'))
+        if not storage:
+            blacklist_device()
+            raise OpenFailed('No storage found for device %s'%(connected_device,))
+        self._main_id = storage[0]['id']
+        self._carda_id = self._cardb_id = None
+        if len(storage) > 1:
+            self._carda_id = storage[1]['id']
+        if len(storage) > 2:
+            self._cardb_id = storage[2]['id']
+
+    @synchronous
+    def get_device_information(self, end_session=True):
+        d = self.dev
+        return (d.friendly_name, d.device_version, d.device_version, '')
+
+    @synchronous
+    def card_prefix(self, end_session=True):
+        ans = [None, None]
+        if self._carda_id is not None:
+            ans[0] = 'mtp:%d:'%self._carda_id
+        if self._cardb_id is not None:
+            ans[1] = 'mtp:%d:'%self._cardb_id
+        return tuple(ans)
+
+    @synchronous
+    def total_space(self, end_session=True):
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['capacity']
+        return tuple(ans)
+
+    @synchronous
+    def free_space(self, end_session=True):
+        self.dev.update_storage_info()
+        ans = [0, 0, 0]
+        for s in self.dev.storage_info:
+            i = {self._main_id:0, self._carda_id:1,
+                    self._cardb_id:2}.get(s['id'], None)
+            if i is not None:
+                ans[i] = s['freespace_bytes']
+        return tuple(ans)
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+    dev = MTP_DEVICE(None)
+    from calibre.devices.scanner import linux_scanner
+    devs = linux_scanner()
+    mtp_devs = dev.detect(devs)
+    dev.open(list(mtp_devs)[0], 'xxx')
+    d = dev.dev
+    print ("Opened device:", dev.get_gui_name())
+    print ("Storage info:")
+    pprint(d.storage_info)
+    print("Free space:", dev.free_space())
+    files, errs = d.get_filelist(dev)
+    pprint((len(files), errs))
+    folders, errs = d.get_folderlist()
+    pprint((len(folders), errs))
+

src/calibre/devices/mtp/unix/libmtp.c

@@ -0,0 +1,559 @@
+#define UNICODE
+#include <Python.h>
+
+#include <stdlib.h>
+#include <libmtp.h>
+
+#include "devices.h"
+
+// Macros and utilities
+#define ENSURE_DEV(rval) \
+    if (self->device == NULL) { \
+        PyErr_SetString(PyExc_ValueError, "This device has not been initialized."); \
+        return rval; \
+    }
+
+#define ENSURE_STORAGE(rval) \
+    if (self->device->storage == NULL) { \
+        PyErr_SetString(PyExc_RuntimeError, "The device has no storage information."); \
+        return rval; \
+    }
+
+// Storage types
+#define ST_Undefined            0x0000
+#define ST_FixedROM             0x0001
+#define ST_RemovableROM         0x0002
+#define ST_FixedRAM             0x0003
+#define ST_RemovableRAM         0x0004
+
+// Storage Access capability
+#define AC_ReadWrite            0x0000
+#define AC_ReadOnly             0x0001
+#define AC_ReadOnly_with_Object_Deletion    0x0002
+
+typedef struct {
+    PyObject *obj;
+    PyThreadState *state;
+} ProgressCallback;
+
+static int report_progress(uint64_t const sent, uint64_t const total, void const *const data) {
+    PyObject *res;
+    ProgressCallback *cb;
+
+    cb = (ProgressCallback *)data;
+    if (cb->obj != NULL) {
+        PyEval_RestoreThread(cb->state);
+        res = PyObject_CallMethod(cb->obj, "report_progress", "KK", sent, total);
+        Py_XDECREF(res);
+        cb->state = PyEval_SaveThread();
+    }
+    return 0;
+}
+
+static void dump_errorstack(LIBMTP_mtpdevice_t *dev, PyObject *list) {
+    LIBMTP_error_t *stack;
+    PyObject *err;
+
+    for(stack = LIBMTP_Get_Errorstack(dev); stack != NULL; stack=stack->next) {
+        err = Py_BuildValue("Is", stack->errornumber, stack->error_text);
+        if (err == NULL) break;
+        PyList_Append(list, err);
+        Py_DECREF(err);
+    }
+
+    LIBMTP_Clear_Errorstack(dev);
+}
+
+// }}}
+
+// Device object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    LIBMTP_mtpdevice_t *device;
+    PyObject *ids;
+    PyObject *friendly_name;
+    PyObject *manufacturer_name;
+    PyObject *model_name;
+    PyObject *serial_number;
+    PyObject *device_version;
+
+} libmtp_Device;
+
+// Device.__init__() {{{
+static void
+libmtp_Device_dealloc(libmtp_Device* self)
+{
+    if (self->device != NULL) LIBMTP_Release_Device(self->device);
+    self->device = NULL;
+
+    Py_XDECREF(self->ids); self->ids = NULL;
+    Py_XDECREF(self->friendly_name); self->friendly_name = NULL;
+    Py_XDECREF(self->manufacturer_name); self->manufacturer_name = NULL;
+    Py_XDECREF(self->model_name); self->model_name = NULL;
+    Py_XDECREF(self->serial_number); self->serial_number = NULL;
+    Py_XDECREF(self->device_version); self->device_version = NULL;
+
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static int
+libmtp_Device_init(libmtp_Device *self, PyObject *args, PyObject *kwds)
+{
+    int busnum, devnum, vendor_id, product_id;
+    PyObject *usb_serialnum;
+    char *vendor, *product, *friendly_name, *manufacturer_name, *model_name, *serial_number, *device_version;
+    LIBMTP_raw_device_t rawdev;
+    LIBMTP_mtpdevice_t *dev;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiiissO", &busnum, &devnum, &vendor_id, &product_id, &vendor, &product, &usb_serialnum)) return -1;
+
+    if (devnum < 0 || devnum > 255 || busnum < 0) { PyErr_SetString(PyExc_TypeError, "Invalid busnum/devnum"); return -1; }
+
+    self->ids = Py_BuildValue("iiiiO", busnum, devnum, vendor_id, product_id, usb_serialnum);
+    if (self->ids == NULL) return -1;
+
+    rawdev.bus_location = (uint32_t)busnum;
+    rawdev.devnum = (uint8_t)devnum;
+    rawdev.device_entry.vendor = vendor;
+    rawdev.device_entry.product = product;
+    rawdev.device_entry.vendor_id = vendor_id;
+    rawdev.device_entry.product_id = product_id;
+    rawdev.device_entry.device_flags = 0x00000000U;
+
+    Py_BEGIN_ALLOW_THREADS;
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == product_id) {
+            rawdev.device_entry.device_flags = calibre_mtp_device_table[i].device_flags;
+        }
+    }
+
+    // Note that contrary to what the libmtp docs imply, we cannot use
+    // LIBMTP_Open_Raw_Device_Uncached as using it causes file listing to fail
+    dev = LIBMTP_Open_Raw_Device(&rawdev);
+    Py_END_ALLOW_THREADS;
+
+    if (dev == NULL) { 
+        PyErr_SetString(PyExc_ValueError, "Unable to open raw device."); 
+        return -1;
+    }
+
+    self->device = dev;
+
+    Py_BEGIN_ALLOW_THREADS;
+    friendly_name = LIBMTP_Get_Friendlyname(self->device);
+    manufacturer_name = LIBMTP_Get_Manufacturername(self->device);
+    model_name = LIBMTP_Get_Modelname(self->device);
+    serial_number = LIBMTP_Get_Serialnumber(self->device);
+    device_version = LIBMTP_Get_Deviceversion(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (friendly_name != NULL) {
+        self->friendly_name = PyUnicode_FromString(friendly_name);
+        free(friendly_name);
+    }
+    if (self->friendly_name == NULL) { self->friendly_name = Py_None; Py_INCREF(Py_None); }
+
+    if (manufacturer_name != NULL) {
+        self->manufacturer_name = PyUnicode_FromString(manufacturer_name);
+        free(manufacturer_name);
+    }
+    if (self->manufacturer_name == NULL) { self->manufacturer_name = Py_None; Py_INCREF(Py_None); }
+
+    if (model_name != NULL) {
+        self->model_name = PyUnicode_FromString(model_name);
+        free(model_name);
+    }
+    if (self->model_name == NULL) { self->model_name = Py_None; Py_INCREF(Py_None); }
+
+    if (serial_number != NULL) {
+        self->serial_number = PyUnicode_FromString(serial_number);
+        free(serial_number);
+    }
+    if (self->serial_number == NULL) { self->serial_number = Py_None; Py_INCREF(Py_None); }
+
+    if (device_version != NULL) {
+        self->device_version = PyUnicode_FromString(device_version);
+        free(device_version);
+    }
+    if (self->device_version == NULL) { self->device_version = Py_None; Py_INCREF(Py_None); }
+
+    return 0;
+}
+// }}}
+
+// Device.friendly_name {{{
+static PyObject *
+libmtp_Device_friendly_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->friendly_name); return self->friendly_name;
+} // }}}
+
+// Device.manufacturer_name {{{
+static PyObject *
+libmtp_Device_manufacturer_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->manufacturer_name); return self->manufacturer_name;
+} // }}}
+
+// Device.model_name {{{
+static PyObject *
+libmtp_Device_model_name(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->model_name); return self->model_name;
+} // }}}
+
+// Device.serial_number {{{
+static PyObject *
+libmtp_Device_serial_number(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->serial_number); return self->serial_number;
+} // }}}
+
+// Device.device_version {{{
+static PyObject *
+libmtp_Device_device_version(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->device_version); return self->device_version;
+} // }}}
+
+// Device.ids {{{
+static PyObject *
+libmtp_Device_ids(libmtp_Device *self, void *closure) {
+    Py_INCREF(self->ids); return self->ids;
+} // }}}
+
+// Device.update_storage_info() {{{
+static PyObject*
+libmtp_Device_update_storage_info(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    ENSURE_DEV(NULL);
+    if (LIBMTP_Get_Storage(self->device, LIBMTP_STORAGE_SORTBY_NOTSORTED) < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "Failed to get storage infor for device.");
+        return NULL;
+    }
+    Py_RETURN_NONE;
+}
+// }}}
+
+// Device.storage_info {{{
+static PyObject *
+libmtp_Device_storage_info(libmtp_Device *self, void *closure) {
+    PyObject *ans, *loc;
+    LIBMTP_devicestorage_t *storage;
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    if (ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    for (storage = self->device->storage; storage != NULL; storage = storage->next) {
+        // Ignore read only storage
+        if (storage->StorageType == ST_FixedROM || storage->StorageType == ST_RemovableROM) continue;
+        // Storage IDs with the lower 16 bits 0x0000 are not supposed to be
+        // writeable.
+        if ((storage->id & 0x0000FFFFU) == 0x00000000U) continue;
+        // Also check the access capability to avoid e.g. deletable only storages
+        if (storage->AccessCapability == AC_ReadOnly || storage->AccessCapability == AC_ReadOnly_with_Object_Deletion) continue;
+
+        loc = Py_BuildValue("{s:k,s:O,s:K,s:K,s:K,s:s,s:s}", 
+                "id", storage->id, 
+                "removable", ((storage->StorageType == ST_RemovableRAM) ? Py_True : Py_False),
+                "capacity", storage->MaxCapacity,
+                "freespace_bytes", storage->FreeSpaceInBytes,
+                "freespace_objects", storage->FreeSpaceInObjects,
+                "storage_desc", storage->StorageDescription,
+                "volume_id", storage->VolumeIdentifier
+        );
+
+        if (loc == NULL) return NULL; 
+        if (PyList_Append(ans, loc) != 0) return NULL;
+        Py_DECREF(loc);
+
+    }
+
+    return ans;
+} // }}}
+
+// Device.get_filelist {{{
+static PyObject *
+libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *fo, *callback = NULL, *errs;
+    ProgressCallback cb;
+    LIBMTP_file_t *f, *tf;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+
+    if (!PyArg_ParseTuple(args, "|O", &callback)) return NULL;
+    cb.obj = callback;
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (ans == NULL || errs == NULL) { PyErr_NoMemory(); return NULL; }
+
+    cb.state = PyEval_SaveThread();
+    tf = LIBMTP_Get_Filelisting_With_Callback(self->device, report_progress, &cb);
+    PyEval_RestoreThread(cb.state);
+
+    if (tf == NULL) { 
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    for (f=tf; f != NULL; f=f->next) {
+        fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}",
+                "id", f->item_id,
+                "parent_id", f->parent_id,
+                "storage_id", f->storage_id,
+                "filename", f->filename,
+                "size", f->filesize,
+                "modtime", f->modificationdate
+        );
+        if (fo == NULL || PyList_Append(ans, fo) != 0) break;
+        Py_DECREF(fo);
+    }
+
+    // Release memory
+    f = tf;
+    while (f != NULL) {
+        tf = f; f = f->next; LIBMTP_destroy_file_t(tf);
+    }
+
+    if (callback != NULL) {
+        // Bug in libmtp where it does not call callback with 100%
+        fo = PyObject_CallMethod(callback, "report_progress", "KK", PyList_Size(ans), PyList_Size(ans));
+        Py_XDECREF(fo);
+    }
+
+    return Py_BuildValue("NN", ans, errs);
+} // }}}
+
+// Device.get_folderlist {{{
+
+int folderiter(LIBMTP_folder_t *f, PyObject *parent) {
+    PyObject *folder, *children;
+
+    children = PyList_New(0);
+    if (children == NULL) { PyErr_NoMemory(); return 1;}
+
+    folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}",
+            "id", f->folder_id,
+            "parent_d", f->parent_id,
+            "storage_id", f->storage_id,
+            "name", f->name,
+            "children", children);
+    if (folder == NULL) return 1;
+    PyList_Append(parent, folder);
+    Py_DECREF(folder);
+
+    if (f->sibling != NULL) {
+        if (folderiter(f->sibling, parent)) return 1;
+    }
+
+    if (f->child != NULL) {
+        if (folderiter(f->child, children)) return 1;
+    }
+
+    return 0;
+}
+
+static PyObject *
+libmtp_Device_get_folderlist(libmtp_Device *self, PyObject *args, PyObject *kwargs) {
+    PyObject *ans, *errs;
+    LIBMTP_folder_t *f;
+
+    ENSURE_DEV(NULL); ENSURE_STORAGE(NULL);
+
+    ans = PyList_New(0);
+    errs = PyList_New(0);
+    if (errs == NULL || ans == NULL) { PyErr_NoMemory(); return NULL; }
+
+    Py_BEGIN_ALLOW_THREADS;
+    f = LIBMTP_Get_Folder_List(self->device);
+    Py_END_ALLOW_THREADS;
+
+    if (f == NULL) {
+        dump_errorstack(self->device, errs);
+        return Py_BuildValue("NN", ans, errs);
+    }
+
+    if (folderiter(f, ans)) return NULL;
+    LIBMTP_destroy_folder_t(f);
+
+    return Py_BuildValue("NN", ans, errs);
+
+} // }}}
+
+static PyMethodDef libmtp_Device_methods[] = {
+    {"update_storage_info", (PyCFunction)libmtp_Device_update_storage_info, METH_VARARGS,
+     "update_storage_info() -> Reread the storage info from the device (total, space, free space, storage locations, etc.)"
+    },
+
+    {"get_filelist", (PyCFunction)libmtp_Device_get_filelist, METH_VARARGS,
+     "get_filelist(callback=None) -> Get the list of files on the device. callback must be an object that has a method named 'report_progress(current, total)'. Returns files, errors."
+    },
+
+    {"get_folderlist", (PyCFunction)libmtp_Device_get_folderlist, METH_VARARGS,
+     "get_folderlist() -> Get the list of folders on the device. Returns files, erros."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef libmtp_Device_getsetters[] = {
+    {(char *)"friendly_name", 
+     (getter)libmtp_Device_friendly_name, NULL,
+     (char *)"The friendly name of this device, can be None.",
+     NULL},
+
+    {(char *)"manufacturer_name", 
+     (getter)libmtp_Device_manufacturer_name, NULL,
+     (char *)"The manufacturer name of this device, can be None.",
+     NULL},
+
+    {(char *)"model_name", 
+     (getter)libmtp_Device_model_name, NULL,
+     (char *)"The model name of this device, can be None.",
+     NULL},
+
+    {(char *)"serial_number", 
+     (getter)libmtp_Device_serial_number, NULL,
+     (char *)"The serial number of this device, can be None.",
+     NULL},
+
+    {(char *)"device_version", 
+     (getter)libmtp_Device_device_version, NULL,
+     (char *)"The device version of this device, can be None.",
+     NULL},
+
+    {(char *)"ids", 
+     (getter)libmtp_Device_ids, NULL,
+     (char *)"The ids of the device (busnum, devnum, vendor_id, product_id, usb_serialnum)",
+     NULL},
+
+    {(char *)"storage_info",
+     (getter)libmtp_Device_storage_info, NULL,
+     (char *)"Information about the storage locations on the device. Returns a list of dictionaries where each dictionary corresponds to the LIBMTP_devicestorage_struct.",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject libmtp_DeviceType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "libmtp.Device",            /*tp_name*/
+    sizeof(libmtp_Device),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)libmtp_Device_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Device",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    libmtp_Device_methods,             /* tp_methods */
+    0,             /* tp_members */
+    libmtp_Device_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    (initproc)libmtp_Device_init,      /* tp_init */
+    0,                         /* tp_alloc */
+    0,                 /* tp_new */
+}; // }}}
+
+// }}} End Device object definition
+
+static PyObject *
+libmtp_set_debug_level(PyObject *self, PyObject *args) {
+    int level;
+    if (!PyArg_ParseTuple(args, "i", &level)) return NULL;
+    LIBMTP_Set_Debug(level);
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+libmtp_is_mtp_device(PyObject *self, PyObject *args) {
+    int busnum, devnum, vendor_id, prod_id, ans = 0;
+    size_t i;
+
+    if (!PyArg_ParseTuple(args, "iiii", &busnum, &devnum, &vendor_id, &prod_id)) return NULL;
+
+    for (i = 0; ; i++) {
+        if (calibre_mtp_device_table[i].vendor == NULL && calibre_mtp_device_table[i].product == NULL && calibre_mtp_device_table[i].vendor_id == 0xffff) break;
+        if (calibre_mtp_device_table[i].vendor_id == vendor_id && calibre_mtp_device_table[i].product_id == prod_id) {
+            Py_RETURN_TRUE;
+        }
+    }
+
+    /*
+     * LIBMTP_Check_Specific_Device does not seem to work at least on my linux
+     * system. Need to investigate why later. Most devices are in the device
+     * table so this is not terribly important.
+     */
+    /* LIBMTP_Set_Debug(LIBMTP_DEBUG_ALL); */
+    /* printf("Calling check: %d %d\n", busnum, devnum); */
+    Py_BEGIN_ALLOW_THREADS;
+    ans = LIBMTP_Check_Specific_Device(busnum, devnum);
+    Py_END_ALLOW_THREADS;
+
+    if (ans) Py_RETURN_TRUE;
+
+    Py_RETURN_FALSE;
+
+}
+
+static PyMethodDef libmtp_methods[] = {
+    {"set_debug_level", libmtp_set_debug_level, METH_VARARGS,
+        "set_debug_level(level)\n\nSet the debug level bit mask, see LIBMTP_DEBUG_* constants."
+    },
+
+    {"is_mtp_device", libmtp_is_mtp_device, METH_VARARGS,
+        "is_mtp_device(busnum, devnum, vendor_id, prod_id)\n\nReturn True if the device is recognized as an MTP device by its vendor/product ids. If it is not recognized a probe is done and True returned if the probe succeeds. Note that probing can cause some devices to malfunction, and it is not very reliable, which is why we prefer to use the device database."
+    },
+
+    {NULL, NULL, 0, NULL}
+};
+
+
+PyMODINIT_FUNC
+initlibmtp(void) {
+    PyObject *m;
+
+    libmtp_DeviceType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&libmtp_DeviceType) < 0)
+        return;
+    
+    m = Py_InitModule3("libmtp", libmtp_methods, "Interface to libmtp.");
+    if (m == NULL) return;
+
+    LIBMTP_Init();
+    LIBMTP_Set_Debug(LIBMTP_DEBUG_NONE);
+
+    Py_INCREF(&libmtp_DeviceType);
+    PyModule_AddObject(m, "Device", (PyObject *)&libmtp_DeviceType);
+
+    PyModule_AddStringMacro(m, LIBMTP_VERSION_STRING);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_NONE);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PTP);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_PLST);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_USB);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_DATA);
+    PyModule_AddIntMacro(m, LIBMTP_DEBUG_ALL);
+}

src/calibre/devices/mtp/unix/upstream/device-flags.h

@@ -0,0 +1,329 @@
+/**
+ * \file device-flags.h
+ * Special device flags to deal with bugs in specific devices.
+ *
+ * Copyright (C) 2005-2007 Richard A. Low <richard@wentnet.com>
+ * Copyright (C) 2005-2012 Linus Walleij <triad@df.lth.se>
+ * Copyright (C) 2006-2007 Marcus Meissner
+ * Copyright (C) 2007 Ted Bullock
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * This file is supposed to be included by both libmtp and libgphoto2.
+ */
+
+/**
+ * These flags are used to indicate if some or other
+ * device need special treatment. These should be possible
+ * to concatenate using logical OR so please use one bit per
+ * feature and lets pray we don't need more than 32 bits...
+ */
+#define DEVICE_FLAG_NONE 0x00000000
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList is broken
+ * in the sense that it won't return properly formatted metadata
+ * for ALL files on the device when you request an object
+ * property list for object 0xFFFFFFFF with parameter 3 likewise
+ * set to 0xFFFFFFFF. Compare to
+ * DEVICE_FLAG_BROKEN_MTPGETOBJECTPROPLIST which only signify
+ * that it's broken when getting metadata for a SINGLE object.
+ * A typical way the implementation may be broken is that it
+ * may not return a proper count of the objects, and sometimes
+ * (like on the ZENs) objects are simply missing from the list
+ * if you use this. Sometimes it has been used incorrectly to
+ * mask bugs in the code (like handling transactions of data
+ * with size given to -1 (0xFFFFFFFFU), in that case please
+ * help us remove it now the code is fixed. Sometimes this is
+ * used because getting all the objects is just too slow and
+ * the USB transaction will time out if you use this command.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST_ALL 0x00000001
+/**
+ * This means that under Linux, another kernel module may
+ * be using this device's USB interface, so we need to detach
+ * it if it is. Typically this is on dual-mode devices that
+ * will present both an MTP compliant interface and device
+ * descriptor *and* a USB mass storage interface. If the USB
+ * mass storage interface is in use, other apps (like our
+ * userspace libmtp through libusb access path) cannot get in
+ * and get cosy with it. So we can remove the offending
+ * application. Typically this means you have to run the program
+ * as root as well.
+ */
+#define DEVICE_FLAG_UNLOAD_DRIVER 0x00000002
+/**
+ * This means that the PTP_OC_MTP_GetObjPropList (9805)
+ * is broken in some way, either it doesn't work at all
+ * (as for Android devices) or it won't properly return all
+ * object properties if parameter 3 is set to 0xFFFFFFFFU.
+ */
+#define DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST 0x00000004
+/**
+ * This means the device doesn't send zero packets to indicate
+ * end of transfer when the transfer boundary occurs at a
+ * multiple of 64 bytes (the USB 1.1 endpoint size). Instead,
+ * exactly one extra byte is sent at the end of the transfer
+ * if the size is an integer multiple of USB 1.1 endpoint size
+ * (64 bytes).
+ *
+ * This behaviour is most probably a workaround due to the fact
+ * that the hardware USB slave controller in the device cannot
+ * handle zero writes at all, and the usage of the USB 1.1
+ * endpoint size is due to the fact that the device will "gear
+ * down" on a USB 1.1 hub, and since 64 bytes is a multiple of
+ * 512 bytes, it will work with USB 1.1 and USB 2.0 alike.
+ */
+#define DEVICE_FLAG_NO_ZERO_READS 0x00000008
+/**
+ * This flag means that the device is prone to forgetting the
+ * OGG container file type, so that libmtp must look at the
+ * filename extensions in order to determine that a file is
+ * actually OGG. This is a clear and present firmware bug, and
+ * while firmware bugs should be fixed in firmware, we like
+ * OGG so much that we back it by introducing this flag.
+ * The error has only been seen on iriver devices. Turning this
+ * flag on won't hurt anything, just that the check against
+ * filename extension will be done for files of "unknown" type.
+ * If the player does not even know (reports) that it supports
+ * ogg even though it does, please use the stronger
+ * OGG_IS_UNKNOWN flag, which will forcedly support ogg on
+ * anything with the .ogg filename extension.
+ */
+#define DEVICE_FLAG_IRIVER_OGG_ALZHEIMER 0x00000010
+/**
+ * This flag indicates a limitation in the filenames a device
+ * can accept - they must be 7 bit (all chars <= 127/0x7F).
+ * It was found first on the Philips Shoqbox, and is a deviation
+ * from the PTP standard which mandates that any unicode chars
+ * may be used for filenames. I guess this is caused by a 7bit-only
+ * filesystem being used intrinsically on the device.
+ */
+#define DEVICE_FLAG_ONLY_7BIT_FILENAMES 0x00000020
+/**
+ * This flag indicates that the device will lock up if you
+ * try to get status of endpoints and/or release the interface
+ * when closing the device. This fixes problems with SanDisk
+ * Sansa devices especially. It may be a side-effect of a
+ * Windows behaviour of never releasing interfaces.
+ */
+#define DEVICE_FLAG_NO_RELEASE_INTERFACE 0x00000040
+/**
+ * This flag was introduced with the advent of Creative ZEN
+ * 8GB. The device sometimes return a broken PTP header
+ * like this: < 1502 0000 0200 01d1 02d1 01d2 >
+ * the latter 6 bytes (representing "code" and "transaction ID")
+ * contain junk. This is breaking the PTP/MTP spec but works
+ * on Windows anyway, probably because the Windows implementation
+ * does not check that these bytes are valid. To interoperate
+ * with devices like this, we need this flag to emulate the
+ * Windows bug. Broken headers has also been found in the
+ * Aricent MTP stack.
+ */
+#define DEVICE_FLAG_IGNORE_HEADER_ERRORS 0x00000080
+/**
+ * The Motorola RAZR2 V8 (others?) has broken set object
+ * proplist causing the metadata setting to fail. (The
+ * set object prop to set individual properties work on
+ * this device, but the metadata is plain ignored on
+ * tracks, though e.g. playlist names can be set.)
+ */
+#define DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST 0x00000100
+/**
+ * The Samsung YP-T10 think Ogg files shall be sent with
+ * the "unknown" (PTP_OFC_Undefined) file type, this gives a
+ * side effect that is a combination of the iRiver Ogg Alzheimer
+ * problem (have to recognized Ogg files on file extension)
+ * and a need to report the Ogg support (the device itself does
+ * not properly claim to support it) and need to set filetype
+ * to unknown when storing Ogg files, even though they're not
+ * actually unknown. Later iRivers seem to need this flag since
+ * they do not report to support OGG even though they actually
+ * do. Often the device supports OGG in USB mass storage mode,
+ * then the firmware simply miss to declare metadata support
+ * for OGG properly.
+ */
+#define DEVICE_FLAG_OGG_IS_UNKNOWN 0x00000200
+/**
+ * The Creative Zen is quite unstable in libmtp but seems to
+ * be better with later firmware versions. However, it still
+ * frequently crashes when setting album art dimensions. This
+ * flag disables setting the dimensions (which seems to make
+ * no difference to how the graphic is displayed).
+ */
+#define DEVICE_FLAG_BROKEN_SET_SAMPLE_DIMENSIONS 0x00000400
+/**
+ * Some devices, particularly SanDisk Sansas, need to always
+ * have their "OS Descriptor" probed in order to work correctly.
+ * This flag provides that extra massage.
+ */
+#define DEVICE_FLAG_ALWAYS_PROBE_DESCRIPTOR 0x00000800
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 1.00 is just a simple playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V1 0x00001000
+/**
+ * Samsung has implimented its own playlist format as a .spl file
+ * stored in the normal file system, rather than a proper mtp
+ * playlist. There are multiple versions of the .spl format
+ * identified by a line in the file: VERSION X.XX
+ * Version 2.00 is playlist but allows DNSe sound settings
+ * to be stored, per playlist.
+ */
+#define DEVICE_FLAG_PLAYLIST_SPL_V2 0x00002000
+/**
+ * The Sansa E250 is know to have this problem which is actually
+ * that the device claims that property PTP_OPC_DateModified
+ * is read/write but will still fail to update it. It can only
+ * be set properly the first time a file is sent.
+ */
+#define DEVICE_FLAG_CANNOT_HANDLE_DATEMODIFIED 0x00004000
+/**
+ * This avoids use of the send object proplist which
+ * is used when creating new objects (not just updating)
+ * The DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST is related
+ * but only concerns the case where the object proplist
+ * is sent in to update an existing object. The Toshiba
+ * Gigabeat MEU202 for example has this problem.
+ */
+#define DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST 0x00008000
+/**
+ * Devices that cannot support reading out battery
+ * level.
+ */
+#define DEVICE_FLAG_BROKEN_BATTERY_LEVEL 0x00010000
+
+/**
+ * Devices that send "ObjectDeleted" events after deletion
+ * of images. (libgphoto2)
+ */
+#define DEVICE_FLAG_DELETE_SENDS_EVENT	0x00020000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE		0x00040000
+
+/**
+ * Cameras that can capture images. (libgphoto2)
+ */
+#define DEVICE_FLAG_CAPTURE_PREVIEW	0x00080000
+
+/**
+ * Nikon broken capture support without proper ObjectAdded events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NIKON_BROKEN_CAPTURE	0x00100000
+
+/**
+ * Broken capture support where cameras do not send CaptureComplete events.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_NO_CAPTURE_COMPLETE		0x00400000
+
+/**
+ * Direct PTP match required.
+ * (libgphoto2)
+ */
+#define DEVICE_FLAG_MATCH_PTP_INTERFACE		0x00800000
+/**
+ * This flag is like DEVICE_FLAG_OGG_IS_UNKNOWN but for FLAC
+ * files instead. Using the unknown filetype for FLAC files.
+ */
+#define DEVICE_FLAG_FLAC_IS_UNKNOWN		0x01000000
+/**
+ * Device needs unique filenames, no two files can be
+ * named the same string.
+ */
+#define DEVICE_FLAG_UNIQUE_FILENAMES		0x02000000
+/**
+ * This flag performs some random magic on the BlackBerry
+ * device to switch from USB mass storage to MTP mode we think.
+ */
+#define DEVICE_FLAG_SWITCH_MODE_BLACKBERRY	0x04000000
+/**
+ * This flag indicates that the device need an extra long
+ * timeout on some operations.
+ */
+#define DEVICE_FLAG_LONG_TIMEOUT		0x08000000
+/**
+ * This flag indicates that the device need an explicit
+ * USB reset after each connection. Some devices don't
+ * like this, so it's not done by default.
+ */
+#define DEVICE_FLAG_FORCE_RESET_ON_CLOSE	0x10000000
+/**
+ * Early Creative Zen (etc) models actually only support
+ * command 9805 (Get object property list) and will hang
+ * if you try to get individual properties of an object.
+ */
+#define DEVICE_FLAG_BROKEN_GET_OBJECT_PROPVAL	0x20000000
+/**
+ * It seems that some devices return an bad data when
+ * using the GetObjectInfo operation. So in these cases
+ * we prefer to override the PTP-compatible object infos
+ * with the MTP property list.
+ *
+ * For example Some Samsung Galaxy S devices contain an MTP
+ * stack that present the ObjectInfo in 64 bit instead of
+ * 32 bit.
+ */
+#define DEVICE_FLAG_PROPLIST_OVERRIDES_OI	0x40000000
+
+/**
+ * All these bug flags need to be set on SONY NWZ Walkman
+ * players, and will be autodetected on unknown devices
+ * by detecting the vendor extension descriptor "sony.net"
+ */
+#define DEVICE_FLAGS_SONY_NWZ_BUGS \
+  (DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_UNIQUE_FILENAMES | \
+   DEVICE_FLAG_FORCE_RESET_ON_CLOSE )
+/**
+ * All these bug flags need to be set on Android devices,
+ * they claim to support MTP operations they actually
+ * cannot handle, especially 9805 (Get object property list).
+ * These are auto-assigned to devices reporting
+ * "android.com" in their device extension descriptor.
+ */
+#define DEVICE_FLAGS_ANDROID_BUGS \
+  (DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST | \
+   DEVICE_FLAG_BROKEN_SET_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_UNLOAD_DRIVER | \
+   DEVICE_FLAG_LONG_TIMEOUT )
+/**
+ * All these bug flags appear on a number of SonyEricsson
+ * devices including Android devices not using the stock
+ * Android 4.0+ (Ice Cream Sandwich) MTP stack. It is highly
+ * supected that these bugs comes from an MTP implementation
+ * from Aricent, so it is called the Aricent bug flags as a
+ * shorthand. Especially the header errors that need to be
+ * ignored is typical for this stack.
+ *
+ * After some guesswork we auto-assign these bug flags to
+ * devices that present the "microsoft.com/WPDNA", and
+ * "sonyericsson.com/SE" but NOT the "android.com"
+ * descriptor.
+ */
+#define DEVICE_FLAGS_ARICENT_BUGS \
+  (DEVICE_FLAG_IGNORE_HEADER_ERRORS | \
+   DEVICE_FLAG_BROKEN_SEND_OBJECT_PROPLIST | \
+   DEVICE_FLAG_BROKEN_MTPGETOBJPROPLIST )

src/calibre/devices/mtp/unix/upstream/update.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+MP = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/music-players.h;hb=HEAD'
+DF = 'http://libmtp.git.sourceforge.net/git/gitweb.cgi?p=libmtp/libmtp;a=blob_plain;f=src/device-flags.h;hb=HEAD'
+
+import urllib, os, shutil
+
+base = os.path.dirname(os.path.abspath(__file__))
+
+for url, fname in [(MP, 'music-players.h'), (DF, 'device-flags.h')]:
+    with open(os.path.join(base, fname), 'wb') as f:
+        shutil.copyfileobj(urllib.urlopen(url), f)
+

src/calibre/devices/scanner.py

@@ -7,6 +7,7 @@ manner.
 
 import sys, os, re
 from threading import RLock
+from collections import namedtuple
 
 from calibre import prints, as_unicode
 from calibre.constants import iswindows, isosx, plugins, islinux, isfreebsd
@@ -107,6 +108,15 @@ class WinPNPScanner(object):
 
 win_pnp_drives = WinPNPScanner()
 
+_USBDevice = namedtuple('USBDevice',
+    'vendor_id product_id bcd manufacturer product serial')
+
+class USBDevice(_USBDevice):
+
+    def __init__(self, *args, **kwargs):
+        _USBDevice.__init__(self, *args, **kwargs)
+        self.busnum = self.devnum = -1
+
 class LinuxScanner(object):
 
     SYSFS_PATH = os.environ.get('SYSFS_PATH', '/sys')
@@ -122,6 +132,10 @@ class LinuxScanner(object):
         if not self.ok:
             raise RuntimeError('DeviceScanner requires the /sys filesystem to work.')
 
+        def read(f):
+            with open(f, 'rb') as s:
+                return s.read().strip()
+
         for x in os.listdir(self.base):
             base = os.path.join(self.base, x)
             ven = os.path.join(base, 'idVendor')
@@ -132,31 +146,46 @@ class LinuxScanner(object):
             prod_string = os.path.join(base, 'product')
             dev = []
             try:
-                dev.append(int('0x'+open(ven).read().strip(), 16))
+                # Ignore USB HUBs
+                if read(os.path.join(base, 'bDeviceClass')) == b'09':
+                    continue
             except:
                 continue
             try:
-                dev.append(int('0x'+open(prod).read().strip(), 16))
+                dev.append(int(b'0x'+read(ven), 16))
             except:
                 continue
             try:
-                dev.append(int('0x'+open(bcd).read().strip(), 16))
+                dev.append(int(b'0x'+read(prod), 16))
             except:
                 continue
             try:
-                dev.append(open(man).read().strip())
+                dev.append(int(b'0x'+read(bcd), 16))
             except:
-                dev.append('')
+                continue
             try:
-                dev.append(open(prod_string).read().strip())
+                dev.append(read(man))
             except:
-                dev.append('')
+                dev.append(b'')
             try:
-                dev.append(open(serial).read().strip())
+                dev.append(read(prod_string))
             except:
-                dev.append('')
+                dev.append(b'')
+            try:
+                dev.append(read(serial))
+            except:
+                dev.append(b'')
 
-            ans.add(tuple(dev))
+            dev = USBDevice(*dev)
+            try:
+                dev.busnum = int(read(os.path.join(base, 'busnum')))
+            except:
+                pass
+            try:
+                dev.devnum = int(read(os.path.join(base, 'devnum')))
+            except:
+                pass
+            ans.add(dev)
         return ans
 
 class FreeBSDScanner(object):

src/calibre/devices/smart_device_app/driver.py

@@ -17,7 +17,7 @@ from calibre.constants import numeric_version, DEBUG
 from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
                                     InitialConnectionError)
 from calibre.devices.interface import DevicePlugin
-from calibre.devices.usbms.books import Book, BookList
+from calibre.devices.usbms.books import Book, CollectionsBookList
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.usbms.driver import USBMS
 from calibre.ebooks import BOOK_EXTENSIONS
@@ -107,6 +107,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     }
     reverse_opcodes = dict([(v, k) for k,v in opcodes.iteritems()])
 
+    ALL_BY_TITLE  = _('All by title')
+    ALL_BY_AUTHOR = _('All by author')
 
     EXTRA_CUSTOMIZATION_MESSAGE = [
         _('Enable connections at startup') + ':::<p>' +
@@ -122,6 +124,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             _('Enter the port number the driver is to use if the "fixed port" box is checked') + '</p>',
         _('Print extra debug information') + ':::<p>' +
             _('Check this box if requested when reporting problems') + '</p>',
+        '',
+        _('Comma separated list of metadata fields '
+            'to turn into collections on the device. Possibilities include: ')+\
+                    'series, tags, authors' +\
+            _('. Two special collections are available: %(abt)s:%(abtv)s and %(aba)s:%(abav)s. Add  '
+            'these values to the list to enable them. The collections will be '
+            'given the name provided after the ":" character.')%dict(
+                            abt='abt', abtv=ALL_BY_TITLE, aba='aba', abav=ALL_BY_AUTHOR)
         ]
     EXTRA_CUSTOMIZATION_DEFAULT = [
                 False,
@@ -130,12 +140,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 '',
                 False, '9090',
                 False,
+                '',
+                ''
     ]
     OPT_AUTOSTART               = 0
     OPT_PASSWORD                = 2
     OPT_USE_PORT                = 4
     OPT_PORT_NUMBER             = 5
     OPT_EXTRA_DEBUG             = 6
+    OPT_COLLECTIONS             = 8
 
     def __init__(self, path):
         self.sync_lock = threading.RLock()
@@ -586,7 +599,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     # the device.
                     raise OpenFailed('')
             try:
-                peer = self.device_socket.getpeername()
+                peer = self.device_socket.getpeername()[0]
                 self.connection_attempts[peer] = 0
             except:
                 pass
@@ -659,9 +672,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     def books(self, oncard=None, end_session=True):
         self._debug(oncard)
         if oncard is not None:
-            return BookList(None, None, None)
+            return CollectionsBookList(None, None, None)
         opcode, result = self._call_client('GET_BOOK_COUNT', {})
-        bl = BookList(None, self.PREFIX, self.settings)
+        bl = CollectionsBookList(None, self.PREFIX, self.settings)
         if opcode == 'OK':
             count = result['count']
             for i in range(0, count):
@@ -680,11 +693,23 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def sync_booklists(self, booklists, end_session=True):
-        self._debug()
+        colattrs = [x.strip() for x in
+                self.settings().extra_customization[self.OPT_COLLECTIONS].split(',')]
+        self._debug('collection attributes', colattrs)
+        coldict = {}
+        if colattrs:
+            collections = booklists[0].get_collections(colattrs)
+            for k,v in collections.iteritems():
+                lpaths = []
+                for book in v:
+                    lpaths.append(book.lpath)
+                coldict[k] = lpaths
+
         # If we ever do device_db plugboards, this is where it will go. We will
         # probably need to send two booklists, one with calibre's data that is
         # given back by "books", and one that has been plugboarded.
-        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]) } )
+        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]),
+                                              'collections': coldict} )
         for i,book in enumerate(booklists[0]):
             if not self._metadata_already_on_device(book):
                 self._set_known_metadata(book)

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -73,7 +73,10 @@ class HTMLTOCAdder(object):
                 if (hasattr(item.data, 'xpath') and
                     XPath('//h:a[@href]')(item.data)):
                     if oeb.spine.index(item) < 0:
+                        if self.position == 'end':
                             oeb.spine.add(item, linear=False)
+                        else:
+                            oeb.spine.insert(0, item, linear=True)
                     return
                 elif has_toc:
                     oeb.guide.remove('toc')

src/calibre/gui2/catalog/catalog_epub_mobi.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>650</width>
-    <height>596</height>
+    <height>603</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -35,7 +35,7 @@
       </size>
      </property>
      <property name="toolTip">
-      <string>Sections to include in catalog.</string>
+      <string>Enabled sections will be included in the generated catalog.</string>
      </property>
      <property name="title">
       <string>Included sections</string>
@@ -107,10 +107,8 @@
       </size>
      </property>
      <property name="toolTip">
-      <string>&lt;p&gt;Default pattern 
+      <string>A regular expression describing genres to be excluded from the generated catalog. Genres are derived from the tags applied to your books.
-\[.+\]
+The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book], and '+', the default tag for a read book.</string>
-excludes tags of the form [tag], 
-e.g., [Project Gutenberg]&lt;/p&gt;</string>
      </property>
      <property name="title">
       <string>Excluded genres</string>
@@ -177,13 +175,23 @@ e.g., [Project Gutenberg]&lt;/p&gt;</string>
           </property>
          </widget>
         </item>
+        <item>
+         <widget class="QToolButton" name="reset_exclude_genres_tb">
+          <property name="toolTip">
+           <string>Reset to default</string>
+          </property>
+          <property name="text">
+           <string>...</string>
+          </property>
+         </widget>
+        </item>
        </layout>
       </item>
      </layout>
     </widget>
    </item>
    <item>
-    <widget class="QGroupBox" name="excludedBooks">
+    <widget class="QGroupBox" name="exclusion_rules_gb">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
        <horstretch>0</horstretch>
@@ -197,226 +205,27 @@ e.g., [Project Gutenberg]&lt;/p&gt;</string>
       </size>
      </property>
      <property name="toolTip">
-      <string>Books matching either pattern will not be included in generated catalog. </string>
+      <string>Books matching any of the exclusion rules will be excluded from the generated catalog. </string>
      </property>
      <property name="title">
       <string>Excluded books</string>
      </property>
-     <layout class="QFormLayout" name="formLayout">
-      <item row="0" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="horizontalLayout_3">
-        <item>
-         <widget class="QLabel" name="label_2">
-          <property name="sizePolicy">
-           <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
-            <horstretch>0</horstretch>
-            <verstretch>0</verstretch>
-           </sizepolicy>
-          </property>
-          <property name="minimumSize">
-           <size>
-            <width>175</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="maximumSize">
-           <size>
-            <width>200</width>
-            <height>16777215</height>
-           </size>
-          </property>
-          <property name="text">
-           <string>Tags to &amp;exclude</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-          </property>
-          <property name="wordWrap">
-           <bool>true</bool>
-          </property>
-          <property name="buddy">
-           <cstring>exclude_tags</cstring>
-          </property>
     </widget>
    </item>
    <item>
-         <widget class="QLineEdit" name="exclude_tags">
+    <widget class="QGroupBox" name="prefix_rules_gb">
-          <property name="minimumSize">
-           <size>
-            <width>0</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string>&lt;p&gt;Comma-separated list of tags to exclude.
-Default: ~,Catalog</string>
-          </property>
-         </widget>
-        </item>
-       </layout>
-      </item>
-      <item row="2" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="exclude_spec_hl">
-        <item>
-         <widget class="QLabel" name="label_7">
-          <property name="minimumSize">
-           <size>
-            <width>175</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="maximumSize">
-           <size>
-            <width>200</width>
-            <height>16777215</height>
-           </size>
-          </property>
-          <property name="text">
-           <string>&amp;Column/value</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-          </property>
-          <property name="wordWrap">
-           <bool>true</bool>
-          </property>
-          <property name="buddy">
-           <cstring>exclude_source_field</cstring>
-          </property>
-         </widget>
-        </item>
-        <item>
-         <widget class="QComboBox" name="exclude_source_field">
-          <property name="minimumSize">
-           <size>
-            <width>0</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string>Column containing additional exclusion criteria</string>
-          </property>
-          <property name="sizeAdjustPolicy">
-           <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
-          </property>
-          <property name="minimumContentsLength">
-           <number>18</number>
-          </property>
-         </widget>
-        </item>
-        <item>
-         <widget class="QLineEdit" name="exclude_pattern">
-          <property name="minimumSize">
-           <size>
-            <width>150</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string>Exclusion pattern</string>
-          </property>
-         </widget>
-        </item>
-       </layout>
-      </item>
-     </layout>
-    </widget>
-   </item>
-   <item>
-    <widget class="QGroupBox" name="readBooks">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
        <horstretch>0</horstretch>
        <verstretch>0</verstretch>
       </sizepolicy>
      </property>
-     <property name="minimumSize">
-      <size>
-       <width>0</width>
-       <height>0</height>
-      </size>
-     </property>
      <property name="toolTip">
-      <string>Matching books will be displayed with a check mark</string>
+      <string>The first enabled matching rule will be used to add a prefix to book listings in the generated catalog.</string>
      </property>
      <property name="title">
-      <string>Read books</string>
+      <string>Prefix rules</string>
      </property>
-     <layout class="QFormLayout" name="formLayout_2">
-      <item row="0" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="read_spec_hl">
-        <property name="sizeConstraint">
-         <enum>QLayout::SetDefaultConstraint</enum>
-        </property>
-        <item>
-         <widget class="QLabel" name="label_3">
-          <property name="minimumSize">
-           <size>
-            <width>175</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="maximumSize">
-           <size>
-            <width>200</width>
-            <height>16777215</height>
-           </size>
-          </property>
-          <property name="text">
-           <string>&amp;Column/value</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-          </property>
-          <property name="wordWrap">
-           <bool>true</bool>
-          </property>
-          <property name="buddy">
-           <cstring>read_source_field</cstring>
-          </property>
-         </widget>
-        </item>
-        <item>
-         <widget class="QComboBox" name="read_source_field">
-          <property name="minimumSize">
-           <size>
-            <width>0</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string>Column containing 'read' status</string>
-          </property>
-          <property name="statusTip">
-           <string/>
-          </property>
-          <property name="sizeAdjustPolicy">
-           <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
-          </property>
-          <property name="minimumContentsLength">
-           <number>18</number>
-          </property>
-         </widget>
-        </item>
-        <item>
-         <widget class="QLineEdit" name="read_pattern">
-          <property name="minimumSize">
-           <size>
-            <width>150</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string>'read book' pattern</string>
-          </property>
-          <property name="statusTip">
-           <string/>
-          </property>
-         </widget>
-        </item>
-       </layout>
-      </item>
-     </layout>
     </widget>
    </item>
    <item>
@@ -440,52 +249,10 @@ Default: ~,Catalog</string>
       <property name="fieldGrowthPolicy">
        <enum>QFormLayout::FieldsStayAtSizeHint</enum>
       </property>
-      <item row="1" column="0" colspan="2">
+      <item row="0" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="horizontalLayout_2">
+       <layout class="QHBoxLayout" name="horizontalLayout_5">
         <item>
-         <widget class="QLabel" name="label_5">
+         <widget class="QLabel" name="label_10">
-          <property name="minimumSize">
-           <size>
-            <width>175</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="maximumSize">
-           <size>
-            <width>200</width>
-            <height>16777215</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string/>
-          </property>
-          <property name="text">
-           <string>&amp;Wishlist tag</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-          </property>
-          <property name="wordWrap">
-           <bool>true</bool>
-          </property>
-          <property name="buddy">
-           <cstring>wishlist_tag</cstring>
-          </property>
-         </widget>
-        </item>
-        <item>
-         <widget class="QLineEdit" name="wishlist_tag">
-          <property name="toolTip">
-           <string>Books tagged as Wishlist items will be displayed with an X</string>
-          </property>
-         </widget>
-        </item>
-       </layout>
-      </item>
-      <item row="2" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="horizontalLayout_4">
-        <item>
-         <widget class="QLabel" name="label_4">
           <property name="minimumSize">
            <size>
             <width>175</width>
@@ -499,16 +266,13 @@ Default: ~,Catalog</string>
            </size>
           </property>
           <property name="text">
-           <string>&amp;Thumbnail width</string>
+           <string>&amp;Thumb width</string>
           </property>
           <property name="alignment">
            <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
           </property>
-          <property name="wordWrap">
-           <bool>true</bool>
-          </property>
           <property name="buddy">
-           <cstring>thumb_width</cstring>
+           <cstring>merge_source_field</cstring>
           </property>
          </widget>
         </item>
@@ -520,8 +284,14 @@ Default: ~,Catalog</string>
             <verstretch>0</verstretch>
            </sizepolicy>
           </property>
+          <property name="maximumSize">
+           <size>
+            <width>137</width>
+            <height>16777215</height>
+           </size>
+          </property>
           <property name="toolTip">
-           <string>Size hint for Description cover thumbnails</string>
+           <string>Size hint for cover thumbnails included in Descriptions section.</string>
           </property>
           <property name="suffix">
            <string> inch</string>
@@ -540,38 +310,17 @@ Default: ~,Catalog</string>
           </property>
          </widget>
         </item>
-       </layout>
-      </item>
-      <item row="3" column="0" colspan="2">
-       <layout class="QHBoxLayout" name="horizontalLayout_5">
         <item>
-         <widget class="QLabel" name="label_6">
+         <widget class="Line" name="line_3">
-          <property name="sizePolicy">
+          <property name="orientation">
-           <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+           <enum>Qt::Vertical</enum>
-            <horstretch>0</horstretch>
-            <verstretch>0</verstretch>
-           </sizepolicy>
-          </property>
-          <property name="minimumSize">
-           <size>
-            <width>175</width>
-            <height>0</height>
-           </size>
-          </property>
-          <property name="maximumSize">
-           <size>
-            <width>200</width>
-            <height>16777215</height>
-           </size>
-          </property>
-          <property name="toolTip">
-           <string/>
           </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLabel" name="label_3">
           <property name="text">
-           <string>&amp;Description note</string>
+           <string>&amp;Extra note</string>
-          </property>
-          <property name="alignment">
-           <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
           </property>
           <property name="buddy">
            <cstring>header_note_source_field</cstring>
@@ -592,14 +341,20 @@ Default: ~,Catalog</string>
             <height>0</height>
            </size>
           </property>
+          <property name="maximumSize">
+           <size>
+            <width>16777215</width>
+            <height>16777215</height>
+           </size>
+          </property>
           <property name="toolTip">
-           <string>Custom column source for note to include in Description header area</string>
+           <string>Custom column source for text to include in Description section.</string>
           </property>
          </widget>
         </item>
        </layout>
       </item>
-      <item row="4" column="0" colspan="2">
+      <item row="1" column="0" colspan="2">
        <layout class="QHBoxLayout" name="horizontalLayout_7">
         <item>
          <widget class="QLabel" name="label_9">
@@ -635,7 +390,7 @@ Default: ~,Catalog</string>
            </size>
           </property>
           <property name="toolTip">
-           <string>Additional content merged with Comments during catalog generation</string>
+           <string>Custom column containing additional content to be merged with Comments metadata.</string>
           </property>
          </widget>
         </item>
@@ -649,7 +404,7 @@ Default: ~,Catalog</string>
         <item>
          <widget class="QRadioButton" name="merge_before">
           <property name="toolTip">
-           <string>Merge additional content before Comments</string>
+           <string>Merge additional content before Comments metadata.</string>
           </property>
           <property name="text">
            <string>&amp;Before</string>
@@ -659,7 +414,7 @@ Default: ~,Catalog</string>
         <item>
          <widget class="QRadioButton" name="merge_after">
           <property name="toolTip">
-           <string>Merge additional content after Comments</string>
+           <string>Merge additional content after Comments metadata.</string>
           </property>
           <property name="text">
            <string>&amp;After</string>
@@ -676,7 +431,7 @@ Default: ~,Catalog</string>
         <item>
          <widget class="QCheckBox" name="include_hr">
           <property name="toolTip">
-           <string>Separate Comments and additional content with horizontal rule</string>
+           <string>Separate Comments metadata and additional content with a horizontal rule.</string>
           </property>
           <property name="text">
            <string>&amp;Separator</string>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -414,7 +414,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
                      and fm[f].get('search_terms', None)
                      and f not in ['formats', 'ondevice']) or
                     (fm[f]['datatype'] in ['int', 'float', 'bool', 'datetime'] and
-                     f not in ['id'])):
+                     f not in ['id', 'timestamp'])):
                 self.all_fields.append(f)
                 self.writable_fields.append(f)
             if fm[f]['datatype'] == 'composite':

src/calibre/gui2/tag_browser/model.py

@@ -425,7 +425,8 @@ class TagsModel(QAbstractItemModel): # {{{
             last_idx = -collapse
             category_is_hierarchical = not (
                 key in ['authors', 'publisher', 'news', 'formats', 'rating'] or
-                key not in self.db.prefs.get('categories_using_hierarchy', []))
+                key not in self.db.prefs.get('categories_using_hierarchy', []) or
+                config['sort_tags_by'] != 'name')
 
             for idx,tag in enumerate(data[key]):
                 components = None

src/calibre/gui2/viewer/documentview.py

@@ -1036,14 +1036,14 @@ class DocumentView(QWebView): # {{{
         if not self.handle_key_press(event):
             return QWebView.keyPressEvent(self, event)
 
-    def paged_col_scroll(self, forward=True):
+    def paged_col_scroll(self, forward=True, scroll_past_end=True):
         dir = 'next' if forward else 'previous'
         loc = self.document.javascript(
                 'paged_display.%s_col_location()'%dir, typ='int')
         if loc > -1:
             self.document.scroll_to(x=loc, y=0)
             self.manager.scrolled(self.document.scroll_fraction)
-        else:
+        elif scroll_past_end:
             (self.manager.next_document() if forward else
                     self.manager.previous_document())
 
@@ -1059,7 +1059,8 @@ class DocumentView(QWebView): # {{{
                 self.is_auto_repeat_event = False
         elif key == 'Down':
             if self.document.in_paged_mode:
-                self.paged_col_scroll()
+                self.paged_col_scroll(scroll_past_end=not
+                        self.document.line_scrolling_stops_on_pagebreaks)
             else:
                 if (not self.document.line_scrolling_stops_on_pagebreaks and
                         self.document.at_bottom):
@@ -1068,7 +1069,8 @@ class DocumentView(QWebView): # {{{
                     self.scroll_by(y=15)
         elif key == 'Up':
             if self.document.in_paged_mode:
-                self.paged_col_scroll(forward=False)
+                self.paged_col_scroll(forward=False, scroll_past_end=not
+                        self.document.line_scrolling_stops_on_pagebreaks)
             else:
                 if (not self.document.line_scrolling_stops_on_pagebreaks and
                         self.document.at_top):

src/calibre/library/catalogs/epub_mobi.py

@@ -47,27 +47,28 @@ class EPUB_MOBI(CatalogPlugin):
                            "of the conversion process a bug is occurring.\n"
                            "Default: '%default'\n"
                            "Applies to: ePub, MOBI output formats")),
-                   Option('--exclude-book-marker',
-                          default=':',
-                          dest='exclude_book_marker',
-                          action = None,
-                          help=_("field:pattern specifying custom field/contents indicating book should be excluded.\n"
-                          "Default: '%default'\n"
-                          "Applies to ePub, MOBI output formats")),
                    Option('--exclude-genre',
-                          default='\[.+\]',
+                          default='\[.+\]|\+',
                           dest='exclude_genre',
                           action = None,
-                          help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[<tag>]'\n"
+                          help=_("Regex describing tags to exclude as genres.\n"
+                          "Default: '%default' excludes bracketed tags, e.g. '[Project Gutenberg]', and '+', the default tag for read books.\n"
                           "Applies to: ePub, MOBI output formats")),
-                   Option('--exclude-tags',
+
-                          default=('~,'+_('Catalog')),
+                   Option('--exclusion-rules',
-                          dest='exclude_tags',
+                          default="(('Excluded tags','Tags','~,Catalog'),)",
+                          dest='exclusion_rules',
                           action=None,
-                          help=_("Comma-separated list of tag words indicating book should be excluded from output. "
+                          help=_("Specifies the rules used to exclude books from the generated catalog.\n"
-                              "For example: 'skip' will match 'skip this book' and 'Skip will like this'. "
+                          "The model for an exclusion rule is either\n('<rule name>','Tags','<comma-separated list of tags>') or\n"
-                              "Default: '%default'\n"
+                          "('<rule name>','<custom column>','<pattern>').\n"
-                              "Applies to: ePub, MOBI output formats")),
+                          "For example:\n"
+                          "(('Archived books','#status','Archived'),)\n"
+                          "will exclude a book with a value of 'Archived' in the custom column 'status'.\n"
+                          "When multiple rules are defined, all rules will be applied.\n"
+                          "Default: \n" + '"' + '%default' + '"' + "\n"
+                          "Applies to ePub, MOBI output formats")),
+
                    Option('--generate-authors',
                           default=False,
                           dest='generate_authors',
@@ -121,7 +122,7 @@ class EPUB_MOBI(CatalogPlugin):
                           default='::',
                           dest='merge_comments',
                           action = None,
-                          help=_("<custom field>:[before|after]:[True|False] specifying:\n"
+                          help=_("#<custom field>:[before|after]:[True|False] specifying:\n"
                           " <custom field> Custom field containing notes to merge with Comments\n"
                           " [before|after] Placement of notes with respect to Comments\n"
                           " [True|False] - A horizontal rule is inserted between notes and Comments\n"
@@ -134,11 +135,14 @@ class EPUB_MOBI(CatalogPlugin):
                           help=_("Specifies the output profile.  In some cases, an output profile is required to optimize the catalog for the device.  For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n"
                           "Default: '%default'\n"
                           "Applies to: ePub, MOBI output formats")),
-                   Option('--read-book-marker',
+                   Option('--prefix-rules',
-                          default='tag:+',
+                          default="(('Read books','tags','+','\u2713'),('Wishlist items','tags','Wishlist','\u00d7'))",
-                          dest='read_book_marker',
+                          dest='prefix_rules',
                           action=None,
-                          help=_("field:pattern indicating book has been read.\n" "Default: '%default'\n"
+                          help=_("Specifies the rules used to include prefixes indicating read books, wishlist items and other user-specifed prefixes.\n"
+                          "The model for a prefix rule is ('<rule name>','<source field>','<pattern>','<prefix>').\n"
+                          "When multiple rules are defined, the first matching rule will be used.\n"
+                          "Default:\n" + '"' + '%default' + '"' + "\n"
                           "Applies to ePub, MOBI output formats")),
                    Option('--thumb-width',
                           default='1.0',
@@ -148,12 +152,6 @@ class EPUB_MOBI(CatalogPlugin):
                           "Range: 1.0 - 2.0\n"
                           "Default: '%default'\n"
                           "Applies to ePub, MOBI output formats")),
-                   Option('--wishlist-tag',
-                          default='Wishlist',
-                          dest='wishlist_tag',
-                          action = None,
-                          help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n"
-                          "Applies to: ePub, MOBI output formats")),
                           ]
     # }}}
 
@@ -276,6 +274,27 @@ class EPUB_MOBI(CatalogPlugin):
             log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST))
             opts.thumb_width = "1.0"
 
+        # eval prefix_rules if passed from command line
+        if type(opts.prefix_rules) is not tuple:
+            try:
+                opts.prefix_rules = eval(opts.prefix_rules)
+            except:
+                log.error("malformed --prefix-rules: %s" % opts.prefix_rules)
+                raise
+            for rule in opts.prefix_rules:
+                if len(rule) != 4:
+                    log.error("incorrect number of args for --prefix-rules: %s" % repr(rule))
+
+        # eval exclusion_rules if passed from command line
+        if type(opts.exclusion_rules) is not tuple:
+            try:
+                opts.exclusion_rules = eval(opts.exclusion_rules)
+            except:
+                log.error("malformed --exclusion-rules: %s" % opts.exclusion_rules)
+                raise
+            for rule in opts.exclusion_rules:
+                if len(rule) != 3:
+                    log.error("incorrect number of args for --exclusion-rules: %s" % repr(rule))
 
         # Display opts
         keys = opts_dict.keys()
@@ -284,8 +303,9 @@ class EPUB_MOBI(CatalogPlugin):
         for key in keys:
             if key in ['catalog_title','authorClip','connected_kindle','descriptionClip',
                        'exclude_book_marker','exclude_genre','exclude_tags',
+                       'exclusion_rules',
                        'header_note_source_field','merge_comments',
-                       'output_profile','read_book_marker',
+                       'output_profile','prefix_rules','read_book_marker',
                        'search_text','sort_by','sort_descriptions_by_author','sync',
                        'thumb_width','wishlist_tag']:
                 build_log.append("  %s: %s" % (key, repr(opts_dict[key])))

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -72,6 +72,7 @@ class CatalogBuilder(object):
         self.__currentStep = 0.0
         self.__creator = opts.creator
         self.__db = db
+        self.__defaultPrefix = None
         self.__descriptionClip = opts.descriptionClip
         self.__error = []
         self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \
@@ -91,10 +92,9 @@ class CatalogBuilder(object):
         self.__output_profile = None
         self.__playOrder = 1
         self.__plugin = plugin
+        self.__prefixRules = []
         self.__progressInt = 0.0
         self.__progressString = ''
-        f, _, p = opts.read_book_marker.partition(':')
-        self.__read_book_marker = {'field':f, 'pattern':p}
         f, p, hr = self.opts.merge_comments.split(':')
         self.__merge_comments = {'field':f, 'position':p, 'hr':hr}
         self.__reporter = report_progress
@@ -113,6 +113,9 @@ class CatalogBuilder(object):
                 self.__output_profile = profile
                 break
 
+        # Process prefix rules
+        self.processPrefixRules()
+
         # Confirm/create thumbs archive.
         if self.opts.generate_descriptions:
             if not os.path.exists(self.__cache_dir):
@@ -269,6 +272,13 @@ class CatalogBuilder(object):
                 return self.__db
             return property(fget=fget)
         @dynamic_property
+        def defaultPrefix(self):
+            def fget(self):
+                return self.__defaultPrefix
+            def fset(self, val):
+                self.__defaultPrefix = val
+            return property(fget=fget, fset=fset)
+        @dynamic_property
         def descriptionClip(self):
             def fget(self):
                 return self.__descriptionClip
@@ -363,6 +373,13 @@ class CatalogBuilder(object):
                 return self.__plugin
             return property(fget=fget)
         @dynamic_property
+        def prefixRules(self):
+            def fget(self):
+                return self.__prefixRules
+            def fset(self, val):
+                self.__prefixRules = val
+            return property(fget=fget, fset=fset)
+        @dynamic_property
         def progressInt(self):
             def fget(self):
                 return self.__progressInt
@@ -437,27 +454,12 @@ class CatalogBuilder(object):
             return property(fget=fget, fset=fset)
 
         @dynamic_property
-        def MISSING_SYMBOL(self):
-            def fget(self):
-                return self.__output_profile.missing_char
-            return property(fget=fget)
-        @dynamic_property
-        def NOT_READ_SYMBOL(self):
-            def fget(self):
-                return '<span style="color:white">%s</span>' % self.__output_profile.read_char
-            return property(fget=fget)
-        @dynamic_property
         def READING_SYMBOL(self):
             def fget(self):
                 return '<span style="color:black">&#x25b7;</span>' if self.generateForKindle else \
                         '<span style="color:white">+</span>'
             return property(fget=fget)
         @dynamic_property
-        def READ_SYMBOL(self):
-            def fget(self):
-                return self.__output_profile.read_char
-            return property(fget=fget)
-        @dynamic_property
         def FULL_RATING_SYMBOL(self):
             def fget(self):
                 return self.__output_profile.ratings_char
@@ -468,6 +470,7 @@ class CatalogBuilder(object):
                 return self.__output_profile.empty_ratings_char
             return property(fget=fget)
         @dynamic_property
+
         def READ_PROGRESS_SYMBOL(self):
             def fget(self):
                 return "&#9642;" if self.generateForKindle else '+'
@@ -655,14 +658,30 @@ Author '{0}':
 
         # Merge opts.exclude_tags with opts.search_text
         # Updated to use exact match syntax
-        empty_exclude_tags = False if len(self.opts.exclude_tags) else True
+
+        exclude_tags = []
+        for rule in self.opts.exclusion_rules:
+            if rule[1].lower() == 'tags':
+                exclude_tags.extend(rule[2].split(','))
+
+        # Remove dups
+        self.exclude_tags = exclude_tags = list(set(exclude_tags))
+
+        # Report tag exclusions
+        if self.opts.verbose and self.exclude_tags:
+            data = self.db.get_data_as_dict(ids=self.opts.ids)
+            for record in data:
+                matched = list(set(record['tags']) & set(exclude_tags))
+                if matched :
+                    self.opts.log.info("    - %s (Exclusion rule Tags: '%s')" % (record['title'], str(matched[0])))
+
         search_phrase = ''
-        if not empty_exclude_tags:
+        if exclude_tags:
-            exclude_tags = self.opts.exclude_tags.split(',')
             search_terms = []
             for tag in exclude_tags:
                 search_terms.append("tag:=%s" % tag)
             search_phrase = "not (%s)" % " or ".join(search_terms)
+
         # If a list of ids are provided, don't use search_text
         if self.opts.ids:
             self.opts.search_text = search_phrase
@@ -750,7 +769,7 @@ Author '{0}':
             if record['cover']:
                 this_title['cover'] = re.sub('&amp;', '&', record['cover'])
 
-            this_title['read'] = self.discoverReadStatus(record)
+            this_title['prefix'] = self.discoverPrefix(record)
 
             if record['tags']:
                 this_title['tags'] = self.processSpecialTags(record['tags'],
@@ -991,29 +1010,17 @@ Author '{0}':
 
             # Add books
             pBookTag = Tag(soup, "p")
+            pBookTag['class'] = "line_item"
             ptc = 0
 
-            #  book with read|reading|unread symbol or wishlist item
+            pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
-            if self.opts.wishlist_tag in book.get('tags', []):
-                    pBookTag['class'] = "wishlist_item"
-                    pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                    ptc += 1
-            else:
-                if book['read']:
-                    # check mark
-                    pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                elif book['id'] in self.bookmarked_books:
-                    pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                else:
-                    # hidden check mark
-                    pBookTag['class'] = "unread_book"
-                    pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
             ptc += 1
 
+            spanTag = Tag(soup, "span")
+            spanTag['class'] = "entry"
+            stc = 0
+
+
             # Link to book
             aTag = Tag(soup, "a")
             if self.opts.generate_descriptions:
@@ -1026,12 +1033,12 @@ Author '{0}':
             else:
                 formatted_title = self.by_titles_normal_title_template.format(**args).rstrip()
             aTag.insert(0,NavigableString(escape(formatted_title)))
-            pBookTag.insert(ptc, aTag)
+            spanTag.insert(stc, aTag)
-            ptc += 1
+            stc += 1
 
             # Dot
-            pBookTag.insert(ptc, NavigableString(" &middot; "))
+            spanTag.insert(stc, NavigableString(" &middot; "))
-            ptc += 1
+            stc += 1
 
             # Link to author
             emTag = Tag(soup, "em")
@@ -1040,7 +1047,10 @@ Author '{0}':
                 aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author']))
             aTag.insert(0, NavigableString(book['author']))
             emTag.insert(0,aTag)
-            pBookTag.insert(ptc, emTag)
+            spanTag.insert(stc, emTag)
+            stc += 1
+
+            pBookTag.insert(ptc, spanTag)
             ptc += 1
 
             if divRunningTag is not None:
@@ -1172,10 +1182,8 @@ Author '{0}':
                     aTag['href'] = "%s.html#%s_series" % ('BySeries',
                                                     re.sub('\W','',book['series']).lower())
                     aTag.insert(0, book['series'])
-                    #pSeriesTag.insert(0, NavigableString(self.NOT_READ_SYMBOL))
                     pSeriesTag.insert(0, aTag)
                 else:
-                    #pSeriesTag.insert(0,NavigableString(self.NOT_READ_SYMBOL + '%s' % book['series']))
                     pSeriesTag.insert(0,NavigableString('%s' % book['series']))
 
                 if author_count == 1:
@@ -1189,29 +1197,16 @@ Author '{0}':
 
             # Add books
             pBookTag = Tag(soup, "p")
+            pBookTag['class'] = "line_item"
             ptc = 0
 
-            #  book with read|reading|unread symbol or wishlist item
+            pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
-            if self.opts.wishlist_tag in book.get('tags', []):
-                pBookTag['class'] = "wishlist_item"
-                pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                ptc += 1
-            else:
-                if book['read']:
-                    # check mark
-                    pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                elif book['id'] in self.bookmarked_books:
-                    pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                else:
-                    # hidden check mark
-                    pBookTag['class'] = "unread_book"
-                    pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
             ptc += 1
 
+            spanTag = Tag(soup, "span")
+            spanTag['class'] = "entry"
+            stc = 0
+
             aTag = Tag(soup, "a")
             if self.opts.generate_descriptions:
                 aTag['href'] = "book_%d.html" % (int(float(book['id'])))
@@ -1227,7 +1222,9 @@ Author '{0}':
                 non_series_books += 1
             aTag.insert(0,NavigableString(escape(formatted_title)))
 
-            pBookTag.insert(ptc, aTag)
+            spanTag.insert(ptc, aTag)
+            stc += 1
+            pBookTag.insert(ptc, spanTag)
             ptc += 1
 
             if author_count == 1:
@@ -1337,29 +1334,16 @@ Author '{0}':
 
                     # Add books
                     pBookTag = Tag(soup, "p")
+                    pBookTag['class'] = "line_item"
                     ptc = 0
 
-                    #  book with read|reading|unread symbol or wishlist item
+                    pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
-                    if self.opts.wishlist_tag in new_entry.get('tags', []):
-                        pBookTag['class'] = "wishlist_item"
-                        pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                        ptc += 1
-                    else:
-                        if new_entry['read']:
-                            # check mark
-                            pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                            pBookTag['class'] = "read_book"
-                            ptc += 1
-                        elif new_entry['id'] in self.bookmarked_books:
-                            pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                            pBookTag['class'] = "read_book"
-                            ptc += 1
-                        else:
-                            # hidden check mark
-                            pBookTag['class'] = "unread_book"
-                            pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
                     ptc += 1
 
+                    spanTag = Tag(soup, "span")
+                    spanTag['class'] = "entry"
+                    stc = 0
+
                     aTag = Tag(soup, "a")
                     if self.opts.generate_descriptions:
                         aTag['href'] = "book_%d.html" % (int(float(new_entry['id'])))
@@ -1372,7 +1356,10 @@ Author '{0}':
                         formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip()
                         non_series_books += 1
                     aTag.insert(0,NavigableString(escape(formatted_title)))
-                    pBookTag.insert(ptc, aTag)
+                    spanTag.insert(stc, aTag)
+                    stc += 1
+
+                    pBookTag.insert(ptc, spanTag)
                     ptc += 1
 
                     divTag.insert(dtc, pBookTag)
@@ -1393,29 +1380,16 @@ Author '{0}':
                 for new_entry in date_range_list:
                     # Add books
                     pBookTag = Tag(soup, "p")
+                    pBookTag['class'] = "line_item"
                     ptc = 0
 
-                    #  book with read|reading|unread symbol or wishlist item
+                    pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup))
-                    if self.opts.wishlist_tag in new_entry.get('tags', []):
-                        pBookTag['class'] = "wishlist_item"
-                        pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                        ptc += 1
-                    else:
-                        if new_entry['read']:
-                            # check mark
-                            pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                            pBookTag['class'] = "read_book"
-                            ptc += 1
-                        elif new_entry['id'] in self.bookmarked_books:
-                            pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                            pBookTag['class'] = "read_book"
-                            ptc += 1
-                        else:
-                            # hidden check mark
-                            pBookTag['class'] = "unread_book"
-                            pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
                     ptc += 1
 
+                    spanTag = Tag(soup, "span")
+                    spanTag['class'] = "entry"
+                    stc = 0
+
                     aTag = Tag(soup, "a")
                     if self.opts.generate_descriptions:
                         aTag['href'] = "book_%d.html" % (int(float(new_entry['id'])))
@@ -1427,12 +1401,12 @@ Author '{0}':
                     else:
                         formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip()
                     aTag.insert(0,NavigableString(escape(formatted_title)))
-                    pBookTag.insert(ptc, aTag)
+                    spanTag.insert(stc, aTag)
-                    ptc += 1
+                    stc += 1
 
                     # Dot
-                    pBookTag.insert(ptc, NavigableString(" &middot; "))
+                    spanTag.insert(stc, NavigableString(" &middot; "))
-                    ptc += 1
+                    stc += 1
 
                     # Link to author
                     emTag = Tag(soup, "em")
@@ -1441,7 +1415,10 @@ Author '{0}':
                         aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author']))
                     aTag.insert(0, NavigableString(new_entry['author']))
                     emTag.insert(0,aTag)
-                    pBookTag.insert(ptc, emTag)
+                    spanTag.insert(stc, emTag)
+                    stc += 1
+
+                    pBookTag.insert(ptc, spanTag)
                     ptc += 1
 
                     divTag.insert(dtc, pBookTag)
@@ -1712,14 +1689,13 @@ Author '{0}':
 
         self.opts.sort_by = 'series'
 
-        # Merge opts.exclude_tags with opts.search_text
+        # Merge self.exclude_tags with opts.search_text
         # Updated to use exact match syntax
-        empty_exclude_tags = False if len(self.opts.exclude_tags) else True
+
         search_phrase = 'series:true '
-        if not empty_exclude_tags:
+        if self.exclude_tags:
-            exclude_tags = self.opts.exclude_tags.split(',')
             search_terms = []
-            for tag in exclude_tags:
+            for tag in self.exclude_tags:
                 search_terms.append("tag:=%s" % tag)
             search_phrase += "not (%s)" % " or ".join(search_terms)
 
@@ -1799,30 +1775,16 @@ Author '{0}':
 
             # Add books
             pBookTag = Tag(soup, "p")
+            pBookTag['class'] = "line_item"
             ptc = 0
 
-            book['read'] = self.discoverReadStatus(book)
+            book['prefix'] = self.discoverPrefix(book)
+            pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
+            ptc += 1
 
-            #  book with read|reading|unread symbol or wishlist item
+            spanTag = Tag(soup, "span")
-            if self.opts.wishlist_tag in book.get('tags', []):
+            spanTag['class'] = "entry"
-                pBookTag['class'] = "wishlist_item"
+            stc = 0
-                pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                ptc += 1
-            else:
-                if book.get('read', False):
-                    # check mark
-                    pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                elif book['id'] in self.bookmarked_books:
-                    pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                else:
-                    # hidden check mark
-                    pBookTag['class'] = "unread_book"
-                    pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
-                    ptc += 1
 
             aTag = Tag(soup, "a")
             if self.opts.generate_descriptions:
@@ -1838,12 +1800,13 @@ Author '{0}':
             args = self.generateFormatArgs(book)
             formatted_title = self.by_series_title_template.format(**args).rstrip()
             aTag.insert(0,NavigableString(escape(formatted_title)))
-            pBookTag.insert(ptc, aTag)
+
-            ptc += 1
+            spanTag.insert(stc, aTag)
+            stc += 1
 
             # &middot;
-            pBookTag.insert(ptc, NavigableString(' &middot; '))
+            spanTag.insert(stc, NavigableString(' &middot; '))
-            ptc += 1
+            stc += 1
 
             # Link to author
             aTag = Tag(soup, "a")
@@ -1851,7 +1814,10 @@ Author '{0}':
                 aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor",
                                             self.generateAuthorAnchor(escape(' & '.join(book['authors']))))
             aTag.insert(0, NavigableString(' &amp; '.join(book['authors'])))
-            pBookTag.insert(ptc, aTag)
+            spanTag.insert(stc, aTag)
+            stc += 1
+
+            pBookTag.insert(ptc, spanTag)
             ptc += 1
 
             divTag.insert(dtc, pBookTag)
@@ -1905,7 +1871,7 @@ Author '{0}':
                     this_book['author'] = book['author']
                     this_book['title'] = book['title']
                     this_book['author_sort'] = capitalize(book['author_sort'])
-                    this_book['read'] = book['read']
+                    this_book['prefix'] = book['prefix']
                     this_book['tags'] = book['tags']
                     this_book['id'] = book['id']
                     this_book['series'] = book['series']
@@ -3165,37 +3131,51 @@ Author '{0}':
         if not os.path.isdir(images_path):
             os.makedirs(images_path)
 
-    def discoverReadStatus(self, record):
+    def discoverPrefix(self, record):
         '''
-        Given a field:pattern spec, discover if this book marked as read
+        Evaluate conditions for including prefixes in various listings
-
-        if field == tag, scan tags for pattern
-        if custom field, try regex match for pattern
-        This allows maximum flexibility with fields of type
-            datatype bool: #field_name:True
-            datatype text: #field_name:<string>
-            datatype datetime: #field_name:.*
-
         '''
-        # Legacy handling of special 'read' tag
+        def log_prefix_rule_match_info(rule, record):
-        field = self.__read_book_marker['field']
+            self.opts.log.info("     %s %s by %s (Prefix rule '%s': %s:%s)" %
-        pat = self.__read_book_marker['pattern']
+                               (rule['prefix'],record['title'],
-        if field == 'tag' and pat in record['tags']:
+                                record['authors'][0], rule['name'],
-            return True
+                                rule['field'],rule['pattern']))
 
+        # Compare the record to each rule looking for a match
+        for rule in self.prefixRules:
+            # Literal comparison for Tags field
+            if rule['field'].lower() == 'tags':
+                if rule['pattern'].lower() in map(unicode.lower,record['tags']):
+                    if self.opts.verbose:
+                        log_prefix_rule_match_info(rule, record)
+                    return rule['prefix']
+
+            # Regex match for custom field
+            elif rule['field'].startswith('#'):
                 field_contents = self.__db.get_field(record['id'],
-                                    field,
+                                    rule['field'],
                                     index_is_id=True)
-        if field_contents:
+                if field_contents == '':
+                    field_contents = None
+
+                if field_contents is not None:
                     try:
-                if re.search(pat, unicode(field_contents),
+                        if re.search(rule['pattern'], unicode(field_contents),
                                 re.IGNORECASE) is not None:
-                    return True
+                            if self.opts.verbose:
+                                log_prefix_rule_match_info(rule, record)
+                            return rule['prefix']
                     except:
                         # Compiling of pat failed, ignore it
+                        if self.opts.verbose:
+                            self.opts.log.error("pattern failed to compile: %s" % rule['pattern'])
                         pass
+                elif field_contents is None and rule['pattern'] == 'None':
+                    if self.opts.verbose:
+                        log_prefix_rule_match_info(rule, record)
+                    return rule['prefix']
 
-        return False
+        return None
 
     def filterDbTags(self, tags):
         # Remove the special marker tags from the database's tag list,
@@ -3227,9 +3207,13 @@ Author '{0}':
             if tag in self.markerTags:
                 excluded_tags.append(tag)
                 continue
+            try:
                 if re.search(self.opts.exclude_genre, tag):
                     excluded_tags.append(tag)
                     continue
+            except:
+                self.opts.log.error("\tfilterDbTags(): malformed --exclude-genre regex pattern: %s" % self.opts.exclude_genre)
+
             if tag == ' ':
                 continue
 
@@ -3266,6 +3250,20 @@ Author '{0}':
         else:
             return None
 
+    def formatPrefix(self,prefix_char,soup):
+        # Generate the HTML for the prefix portion of the listing
+        spanTag = Tag(soup, "span")
+        if prefix_char is None:
+            spanTag['style'] = "color:white"
+            spanTag.insert(0,NavigableString(self.defaultPrefix))
+            # 2e3a is 'two-em dash', which matches width in Kindle Previewer
+            # too wide in calibre viewer
+            # minimal visual distraction
+            # spanTag.insert(0,NavigableString(u'\u2e3a'))
+        else:
+            spanTag.insert(0,NavigableString(prefix_char))
+        return spanTag
+
     def generateAuthorAnchor(self, author):
         # Strip white space to ''
         return re.sub("\W","", author)
@@ -3359,29 +3357,16 @@ Author '{0}':
 
             # Add books
             pBookTag = Tag(soup, "p")
+            pBookTag['class'] = "line_item"
             ptc = 0
 
-            #  book with read|reading|unread symbol or wishlist item
+            pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup))
-            if self.opts.wishlist_tag in book.get('tags', []):
-                    pBookTag['class'] = "wishlist_item"
-                    pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL))
-                    ptc += 1
-            else:
-                if book['read']:
-                    # check mark
-                    pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                elif book['id'] in self.bookmarked_books:
-                    pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL))
-                    pBookTag['class'] = "read_book"
-                    ptc += 1
-                else:
-                    # hidden check mark
-                    pBookTag['class'] = "unread_book"
-                    pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL))
             ptc += 1
 
+            spanTag = Tag(soup, "span")
+            spanTag['class'] = "entry"
+            stc = 0
+
             # Add the book title
             aTag = Tag(soup, "a")
             if self.opts.generate_descriptions:
@@ -3398,7 +3383,10 @@ Author '{0}':
                 non_series_books += 1
             aTag.insert(0,NavigableString(escape(formatted_title)))
 
-            pBookTag.insert(ptc, aTag)
+            spanTag.insert(stc, aTag)
+            stc += 1
+
+            pBookTag.insert(ptc, spanTag)
             ptc += 1
 
             divTag.insert(dtc, pBookTag)
@@ -3463,11 +3451,9 @@ Author '{0}':
 
         # Author, author_prefix (read|reading|none symbol or missing symbol)
         author = book['author']
-        if self.opts.wishlist_tag in book.get('tags', []):
+
-            author_prefix =  self.MISSING_SYMBOL + " by "
+        if book['prefix']:
-        else:
+            author_prefix = book['prefix'] + " by "
-            if book['read']:
-                author_prefix = self.READ_SYMBOL + " by "
         elif self.opts.connected_kindle and book['id'] in self.bookmarked_books:
             author_prefix = self.READING_SYMBOL + " by "
         else:
@@ -3846,9 +3832,14 @@ Author '{0}':
                 return friendly_tag
 
     def getMarkerTags(self):
-        ''' Return a list of special marker tags to be excluded from genre list '''
+        '''
+        Return a list of special marker tags to be excluded from genre list
+        exclusion_rules = ('name','Tags|#column','[]|pattern')
+        '''
         markerTags = []
-        markerTags.extend(self.opts.exclude_tags.split(','))
+        for rule in self.opts.exclusion_rules:
+            if rule[1].lower() == 'tags':
+                markerTags.extend(rule[2].split(','))
         return markerTags
 
     def letter_or_symbol(self,char):
@@ -4005,38 +3996,79 @@ Author '{0}':
 
         return merged
 
+    def processPrefixRules(self):
+        if self.opts.prefix_rules:
+            # Put the prefix rules into an ordered list of dicts
+            try:
+                for rule in self.opts.prefix_rules:
+                    prefix_rule = {}
+                    prefix_rule['name'] = rule[0]
+                    prefix_rule['field'] = rule[1]
+                    prefix_rule['pattern'] = rule[2]
+                    prefix_rule['prefix'] = rule[3]
+                    self.prefixRules.append(prefix_rule)
+            except:
+                self.opts.log.error("malformed self.opts.prefix_rules: %s" % repr(self.opts.prefix_rules))
+                raise
+            # Use the highest order prefix symbol as default
+            self.defaultPrefix = self.opts.prefix_rules[0][3]
+
     def processExclusions(self, data_set):
         '''
         Remove excluded entries
         '''
-        field, pat = self.opts.exclude_book_marker.split(':')
-        if pat == '':
-            return data_set
         filtered_data_set = []
+        exclusion_pairs = []
+        exclusion_set = []
+        for rule in self.opts.exclusion_rules:
+            if rule[1].startswith('#') and rule[2] != '':
+                field = rule[1]
+                pat = rule[2]
+                exclusion_pairs.append((field,pat))
+            else:
+                continue
+
+        if exclusion_pairs:
             for record in data_set:
+                for exclusion_pair in exclusion_pairs:
+                    field,pat = exclusion_pair
                     field_contents = self.__db.get_field(record['id'],
                                                 field,
                                                 index_is_id=True)
                     if field_contents:
                         if re.search(pat, unicode(field_contents),
                                 re.IGNORECASE) is not None:
-                    continue
+                            if self.opts.verbose:
+                                field_md = self.db.metadata_for_field(field)
+                                self.opts.log.info("    - %s (Exclusion rule '%s': %s:%s)" %
+                                                   (record['title'], field_md['name'], field,pat))
+                            exclusion_set.append(record)
+                            if record in filtered_data_set:
+                                filtered_data_set.remove(record)
+                            break
+                    else:
+                        if (record not in filtered_data_set and
+                            record not in exclusion_set):
                             filtered_data_set.append(record)
-
             return filtered_data_set
+        else:
+            return data_set
 
     def processSpecialTags(self, tags, this_title, opts):
+
         tag_list = []
+
+        try:
             for tag in tags:
                 tag = self.convertHTMLEntities(tag)
                 if re.search(opts.exclude_genre, tag):
                     continue
-            elif self.__read_book_marker['field'] == 'tag' and \
-                    tag == self.__read_book_marker['pattern']:
-                # remove 'read' tag
-                continue
                 else:
                     tag_list.append(tag)
+        except:
+            self.opts.log.error("\tprocessSpecialTags(): malformed --exclude-genre regex pattern: %s" % opts.exclude_genre)
+            return tags
+
         return tag_list
 
     def updateProgressFullStep(self, description):

src/calibre/library/cli.py

@@ -719,6 +719,7 @@ def catalog_option_parser(args):
     def add_plugin_parser_options(fmt, parser, log):
 
         # Fetch the extension-specific CLI options from the plugin
+        # library.catalogs.<format>.py
         plugin = plugin_for_catalog_format(fmt)
         for option in plugin.cli_options:
             if option.action:
@@ -798,6 +799,7 @@ def catalog_option_parser(args):
 def command_catalog(args, dbpath):
     parser, plugin, log = catalog_option_parser(args)
     opts, args = parser.parse_args(sys.argv[1:])
+
     if len(args) < 2:
         parser.print_help()
         print

src/calibre/library/database2.py

@@ -31,7 +31,8 @@ from calibre.ptempfile import (PersistentTemporaryFile,
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
-from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
+from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
+        parse_only_date)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
@@ -2479,6 +2480,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_pubdate(self, id, dt, notify=True, commit=True):
         if dt:
+            if isinstance(dt, basestring):
+                dt = parse_only_date(dt)
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
             self.dirtied([id], commit=False)