Merge from trunk

resources/recipes/jpost.recipe

@@ -1,3 +1,4 @@
+import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class JerusalemPost(BasicNewsRecipe):
@@ -10,8 +11,6 @@ class JerusalemPost(BasicNewsRecipe):
     __author__ = 'Kovid Goyal'
     max_articles_per_feed = 10
     no_stylesheets = True
-    remove_tags_before = {'class':'jp-grid-content'}
-    remove_tags_after = {'id':'body_val'}
 
     feeds =  [ ('Front Page', 'http://www.jpost.com/servlet/Satellite?pagename=JPost/Page/RSS&cid=1123495333346'),
                ('Israel News', 'http://www.jpost.com/servlet/Satellite?pagename=JPost/Page/RSS&cid=1178443463156'),
@@ -20,9 +19,24 @@ class JerusalemPost(BasicNewsRecipe):
                ('Editorials', 'http://www.jpost.com/servlet/Satellite?pagename=JPost/Page/RSS&cid=1123495333211'),
           ]
 
+    remove_tags = [
+            dict(id=lambda x: x and 'ads.' in x),
+            dict(attrs={'class':['printinfo', 'tt1']}),
+            dict(onclick='DoPrint()'),
+            dict(name='input'),
+            ]
+
+    conversion_options = {'linearize_tables':True}
+
     def preprocess_html(self, soup):
-        for x in soup.findAll(name=['form', 'input']):
+        for tag in soup.findAll('form'):
-            x.name = 'div'
+            tag.name = 'div'
-        for x in soup.findAll('body', style=True):
-            del x['style']
         return soup
+
+    def print_version(self, url):
+        m = re.search(r'(ID|id)=(\d+)', url)
+        if m is not None:
+            id_ = m.group(2)
+            return 'http://www.jpost.com/LandedPages/PrintArticle.aspx?id=%s'%id_
+        return url
+

resources/recipes/onionavclub.recipe

@@ -0,0 +1,36 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+'''
+bbc.co.uk
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BBC(BasicNewsRecipe):
+    title          = u'The Onion AV Club'
+    __author__     = 'Stephen Williams'
+    description    = 'Film, Television and Music Reviews'
+    no_stylesheets = True
+    oldest_article        = 2
+    max_articles_per_feed = 100
+
+    keep_only_tags     = [dict(name='div', attrs={'id':'content'})
+                          ]
+
+    remove_tags    = [dict(name='div', attrs={'class':['footer','tools_horizontal']}),
+                      dict(name='div', attrs={'id':['tool_holder','elsewhere_on_avclub']})
+                      ]
+    extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }'
+
+    feeds          = [
+                      ('Interviews', 'http://www.avclub.com/feed/interview/'),
+                      ('AV Club Daily', 'http://www.avclub.com/feed/daily'),
+                      ('Film', 'http://www.avclub.com/feed/film/'),
+                      ('Music', 'http://www.avclub.com/feed/music/'),
+                      ('DVD', 'http://www.avclub.com/feed/dvd/'),
+                      ('Books', 'http://www.avclub.com/feed/books/'),
+                      ('Games', 'http://www.avclub.com/feed/games/'),
+                      ('Interviews', 'http://www.avclub.com/feed/interview/'),
+                    ]

src/calibre/customize/builtins.py

@@ -454,7 +454,7 @@ from calibre.devices.hanvon.driver import N516, EB511, ALEX
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3
 from calibre.devices.sne.driver import SNE
-from calibre.devices.misc import PALMPRE
+from calibre.devices.misc import PALMPRE, KOBO
 
 from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon
 from calibre.library.catalog import CSV_XML, EPUB_MOBI
@@ -536,7 +536,8 @@ plugins += [
     EDGE,
     SNE,
     ALEX,
-    PALMPRE
+    PALMPRE,
+    KOBO,
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/devices/misc.py

@@ -28,3 +28,24 @@ class PALMPRE(USBMS):
 
     EBOOK_DIR_MAIN = 'E-books'
 
+class KOBO(USBMS):
+
+    name = 'Kobo Reader Device Interface'
+    gui_name = 'Kobo Reader'
+    description = _('Communicate with the Kobo Reader')
+    author = 'Kovid Goyal'
+
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['epub', 'pdf']
+
+    VENDOR_ID   = [0x2237]
+    PRODUCT_ID  = [0x4161]
+    BCD         = [0x0110]
+
+    VENDOR_NAME = 'KOBO_INC'
+    WINDOWS_MAIN_MEM = '.KOBOEREADER'
+
+    EBOOK_DIR_MAIN = 'e-books'
+

src/calibre/ebooks/html/input.py

@@ -11,7 +11,7 @@ __docformat__ = 'restructuredtext en'
 Input plugin for HTML or OPF ebooks.
 '''
 
-import os, re, sys, uuid
+import os, re, sys, uuid, tempfile
 from urlparse import urlparse, urlunparse
 from urllib import unquote
 from functools import partial
@@ -272,6 +272,7 @@ class HTMLInput(InputFormatPlugin):
 
     def convert(self, stream, opts, file_ext, log,
                 accelerators):
+        self._is_case_sensitive = None
         basedir = os.getcwd()
         self.opts = opts
 
@@ -290,6 +291,15 @@ class HTMLInput(InputFormatPlugin):
         return create_oebbook(log, stream.name, opts, self,
                 encoding=opts.input_encoding)
 
+    def is_case_sensitive(self, path):
+        if self._is_case_sensitive is not None:
+            return self._is_case_sensitive
+        if not path or not os.path.exists(path):
+            return islinux or isfreebsd
+        self._is_case_sensitive = os.path.exists(path.lower()) \
+                and os.path.exists(path.upper())
+        return self._is_case_sensitive
+
     def create_oebbook(self, htmlpath, basedir, opts, log, mi):
         from calibre.ebooks.conversion.plumber import create_oebbook
         from calibre.ebooks.oeb.base import DirContainer, \
@@ -343,14 +353,16 @@ class HTMLInput(InputFormatPlugin):
 
         self.added_resources = {}
         self.log = log
+        self.log('Normalizing filename cases')
         for path, href in htmlfile_map.items():
-            if not (islinux or isfreebsd):
+            if not self.is_case_sensitive(path):
                 path = path.lower()
             self.added_resources[path] = href
         self.urlnormalize, self.DirContainer = urlnormalize, DirContainer
         self.urldefrag = urldefrag
         self.guess_type, self.BINARY_MIME = guess_type, BINARY_MIME
 
+        self.log('Rewriting HTML links')
         for f in filelist:
             path = f.path
             dpath = os.path.dirname(path)
@@ -415,7 +427,7 @@ class HTMLInput(InputFormatPlugin):
         if os.path.isdir(link):
             self.log.warn(link_, 'is a link to a directory. Ignoring.')
             return link_
-        if not (islinux or isfreebsd):
+        if not self.is_case_sensitive(tempfile.gettempdir()):
             link = link.lower()
         if link not in self.added_resources:
             bhref = os.path.basename(link)

src/calibre/ebooks/oeb/base.py

@@ -294,6 +294,9 @@ def xml2str(root, pretty_print=False, strip_comments=False):
 def xml2unicode(root, pretty_print=False):
     return etree.tostring(root, pretty_print=pretty_print)
 
+def xml2text(elem):
+    return etree.tostring(elem, method='text', encoding=unicode, with_tail=False)
+
 ASCII_CHARS   = set(chr(x) for x in xrange(128))
 UNIBYTE_CHARS = set(chr(x) for x in xrange(256))
 URL_SAFE      = set('ABCDEFGHIJKLMNOPQRSTUVWXYZ'

src/calibre/ebooks/oeb/transforms/split.py

@@ -5,7 +5,7 @@ __docformat__ = 'restructuredtext en'
 
 '''
 Splitting of the XHTML flows. Splitting can happen on page boundaries or can be
-forces at "likely" locations to conform to size limitations. This transform
+forced at "likely" locations to conform to size limitations. This transform
 assumes a prior call to the flatcss transform.
 '''
 
@@ -385,12 +385,18 @@ class FlowSplitter(object):
             raise SplitError(self.item.href, root)
         self.log.debug('\t\t\tSplit point:', split_point.tag, tree.getpath(split_point))
 
-        for t in self.do_split(tree, split_point, before):
+        trees = self.do_split(tree, split_point, before)
+        sizes = [len(tostring(t.getroot())) for t in trees]
+        if min(sizes) < 5*1024:
+            self.log.debug('\t\t\tSplit tree too small')
+            self.split_to_size(tree)
+            return
+
+        for t, size in zip(trees, sizes):
             r = t.getroot()
             if self.is_page_empty(r):
                 continue
-            size = len(tostring(r))
+            elif size <= self.max_flow_size:
-            if size <= self.max_flow_size:
                 self.split_trees.append(t)
                 self.log.debug(
                     '\t\t\tCommitted sub-tree #%d (%d KB)'%(

src/calibre/ebooks/oeb/transforms/structure.py

@@ -11,7 +11,7 @@ import re
 from lxml import etree
 from urlparse import urlparse
 
-from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML
+from calibre.ebooks.oeb.base import XPNSMAP, TOC, XHTML, xml2text
 from calibre.ebooks import ConversionError
 
 def XPath(x):
@@ -79,8 +79,7 @@ class DetectStructure(object):
             page_break_before = 'display: block; page-break-before: always'
             page_break_after = 'display: block; page-break-after: always'
             for item, elem in self.detected_chapters:
-                text = u' '.join([t.strip() for t in elem.xpath('descendant::text()')])
+                text = xml2text(elem).strip()
-                text = text.strip()
                 self.log('\tDetected chapter:', text[:50])
                 if chapter_mark == 'none':
                     continue
@@ -120,8 +119,7 @@ class DetectStructure(object):
                     if frag:
                         href = '#'.join((href, frag))
                     if not self.oeb.toc.has_href(href):
-                        text = u' '.join([t.strip() for t in \
+                        text = xml2text(a)
-                                a.xpath('descendant::text()')])
                         text = text[:100].strip()
                         if not self.oeb.toc.has_text(text):
                             num += 1
@@ -135,7 +133,7 @@ class DetectStructure(object):
 
 
     def elem_to_link(self, item, elem, counter):
-        text = u' '.join([t.strip() for t in elem.xpath('descendant::text()')])
+        text = xml2text(elem)
         text = text[:100].strip()
         id = elem.get('id', 'calibre_toc_%d'%counter)
         elem.set('id', id)

src/calibre/gui2/__init__.py

@@ -193,11 +193,14 @@ def warning_dialog(parent, title, msg, det_msg='', show=False):
         return d.exec_()
     return d
 
-def error_dialog(parent, title, msg, det_msg='', show=False):
+def error_dialog(parent, title, msg, det_msg='', show=False,
+        show_copy_button=True):
     d = MessageBox(QMessageBox.Critical, 'ERROR: '+title, msg, QMessageBox.Ok,
                     parent, det_msg)
     d.setIconPixmap(QPixmap(I('dialog_error.svg')))
     d.setEscapeButton(QMessageBox.Ok)
+    if not show_copy_button:
+        d.cb.setVisible(False)
     if show:
         return d.exec_()
     return d
@@ -218,9 +221,6 @@ def info_dialog(parent, title, msg, det_msg='', show=False):
     return d
 
 
-def qstring_to_unicode(q):
-    return unicode(q)
-
 def human_readable(size):
     """ Convert a size in bytes into a human readable form """
     divisor, suffix = 1, "B"
@@ -380,7 +380,7 @@ class FileIconProvider(QFileIconProvider):
         if fileinfo.isDir():
             key = 'dir'
         else:
-            ext = qstring_to_unicode(fileinfo.completeSuffix()).lower()
+            ext = unicode(fileinfo.completeSuffix()).lower()
             key = self.key_from_ext(ext)
         return self.cached_icon(key)
 

src/calibre/gui2/dialogs/comicconf.py

@@ -6,18 +6,17 @@ __docformat__ = 'restructuredtext en'
 ''''''
 from PyQt4.QtGui import QDialog
 from calibre.gui2.dialogs.comicconf_ui import Ui_Dialog
-from calibre.gui2 import qstring_to_unicode
 from calibre.ebooks.lrf.comic.convert_from import config, PROFILES
 
 def set_conversion_defaults(window):
     d = ComicConf(window)
     d.exec_()
-    
+
 def get_bulk_conversion_options(window):
     d = ComicConf(window, config_defaults=config(None).as_string())
     if d.exec_() == QDialog.Accepted:
         return d.config.parse()
-    
+
 def get_conversion_options(window, defaults, title, author):
     if defaults is None:
         defaults = config(None).as_string()
@@ -26,10 +25,10 @@ def get_conversion_options(window, defaults, title, author):
     if d.exec_() == QDialog.Accepted:
         return d.config.parse(), d.config.src
     return None, None
-    
+
 
 class ComicConf(QDialog, Ui_Dialog):
-    
+
     def __init__(self, window, config_defaults=None, generic=True,
                  title=_('Set defaults for conversion of comics (CBR/CBZ files)')):
         QDialog.__init__(self, window)
@@ -63,12 +62,12 @@ class ComicConf(QDialog, Ui_Dialog):
         self.opt_despeckle.setChecked(opts.despeckle)
         self.opt_wide.setChecked(opts.wide)
         self.opt_right2left.setChecked(opts.right2left)
-        
+
         for opt in self.config.option_set.preferences:
             g = getattr(self, 'opt_'+opt.name, False)
             if opt.help and g:
                 g.setToolTip(opt.help)
-                
+
     def accept(self):
         for opt in self.config.option_set.preferences:
             g = getattr(self, 'opt_'+opt.name, False)
@@ -78,9 +77,9 @@ class ComicConf(QDialog, Ui_Dialog):
             elif hasattr(g, 'value'):
                 val = g.value()
             elif hasattr(g, 'itemText'):
-                val = qstring_to_unicode(g.itemText(g.currentIndex()))
+                val = unicode(g.itemText(g.currentIndex()))
             elif hasattr(g, 'text'):
-                val = qstring_to_unicode(g.text())
+                val = unicode(g.text())
             else:
                 raise Exception('Bad coding')
             self.config.set(opt.name, val)

src/calibre/gui2/dialogs/config/__init__.py

@@ -8,14 +8,14 @@ from PyQt4.Qt import    QDialog, QListWidgetItem, QIcon, \
                         SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \
                         QModelIndex, QAbstractTableModel, \
                         QDialogButtonBox, QTabWidget, QBrush, QLineEdit, \
-                        QProgressDialog, QMessageBox
+                        QProgressDialog
 
 from calibre.constants import iswindows, isosx, preferred_encoding
 from calibre.gui2.dialogs.config.config_ui import Ui_Dialog
 from calibre.gui2.dialogs.config.create_custom_column import CreateCustomColumn
-from calibre.gui2 import qstring_to_unicode, choose_dir, error_dialog, config, \
+from calibre.gui2 import choose_dir, error_dialog, config, \
                          ALL_COLUMNS, NONE, info_dialog, choose_files, \
-                         warning_dialog, ResizableDialog
+                         warning_dialog, ResizableDialog, question_dialog
 from calibre.utils.config import prefs
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.oeb.iterator import is_supported
@@ -648,16 +648,15 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
     def del_custcol(self):
         idx = self.columns.currentRow()
         if idx < 0:
-            self.messagebox(_('You must select a column to delete it'))
+            return error_dialog(self, '', _('You must select a column to delete it'),
-            return
+                    show=True)
-        col = qstring_to_unicode(self.columns.item(idx).data(Qt.UserRole).toString())
+        col = unicode(self.columns.item(idx).data(Qt.UserRole).toString())
         if col not in self.custcols:
-            self.messagebox(_('The selected column is not a custom column'))
+            return error_dialog(self, '',
-            return
+                    _('The selected column is not a custom column'), show=True)
-        ret = self.messagebox(_('Do you really want to delete column %s and all its data')%self.custcols[col]['name'],
+        if not question_dialog(self, _('Are you sure?'),
-                         buttons=QMessageBox.Ok|QMessageBox.Cancel,
+            _('Do you really want to delete column %s and all its data?') %
-                         defaultButton=QMessageBox.Cancel)
+            self.custcols[col]['name']):
-        if ret != QMessageBox.Ok:
             return
         self.columns.item(idx).setCheckState(False)
         self.columns.takeItem(idx)
@@ -759,12 +758,12 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         config['use_roman_numerals_for_series_number'] = bool(self.roman_numerals.isChecked())
         config['new_version_notification'] = bool(self.new_version_notification.isChecked())
         prefs['network_timeout'] = int(self.timeout.value())
-        path = qstring_to_unicode(self.location.text())
+        path = unicode(self.location.text())
         input_cols = [unicode(self.input_order.item(i).data(Qt.UserRole).toString()) for i in range(self.input_order.count())]
         prefs['input_format_order'] = input_cols
 
         ####### Now deal with changes to columns
-        cols = [qstring_to_unicode(self.columns.item(i).data(Qt.UserRole).toString())\
+        cols = [unicode(self.columns.item(i).data(Qt.UserRole).toString())\
                  for i in range(self.columns.count()) \
                     if self.columns.item(i).checkState()==Qt.Checked]
         if not cols:
@@ -829,15 +828,13 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         else:
             self.database_location = os.path.abspath(path)
             if must_restart:
-                self.messagebox(_('The changes you made require that Calibre be restarted. Please restart as soon as practical.'))
+                warning_dialog(self, _('Must restart'),
+                        _('The changes you made require that Calibre be '
+                            'restarted. Please restart as soon as practical.'),
+                        show=True)
                 self.parent.must_restart_before_config = True
             QDialog.accept(self)
 
-    # might want to substitute the standard calibre box. However, the copy_to_clipboard
-    # functionality has no purpose, so ???
-    def messagebox(self, m, buttons=QMessageBox.Ok, defaultButton=QMessageBox.Ok):
-                return QMessageBox.critical(None,'Calibre configuration', m, buttons, defaultButton)
-
 class VacThread(QThread):
 
     def __init__(self, parent, db):

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -3,25 +3,45 @@ __copyright__ = '2010, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Dialog to create a new custom column'''
 
+from functools import partial
+
 from PyQt4.QtCore import SIGNAL
 from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant
+
 from calibre.gui2.dialogs.config.create_custom_column_ui import Ui_QCreateCustomColumn
+from calibre.gui2 import error_dialog
 
 class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
+
     column_types = {
-                    0:{'datatype':'text', 'text':_('Text, column shown in tags browser'), 'is_multiple':False},
+                    0:{'datatype':'text',
-                    1:{'datatype':'*text', 'text':_('Comma separated text, shown in tags browser'), 'is_multiple':True},
+                        'text':_('Text, column shown in the tag browser'),
-                    2:{'datatype':'comments', 'text':_('Text, column not shown in tags browser'), 'is_multiple':False},
+                        'is_multiple':False},
-                    3:{'datatype':'datetime', 'text':_('Date'), 'is_multiple':False},
+                    1:{'datatype':'*text',
-                    4:{'datatype':'float', 'text':_('Float'), 'is_multiple':False},
+                        'text':_('Comma separated text, like tags, shown in the tag browser'),
-                    5:{'datatype':'int', 'text':_('Integer'), 'is_multiple':False},
+                        'is_multiple':True},
-                    6:{'datatype':'rating', 'text':_('Rating (stars)'), 'is_multiple':False},
+                    2:{'datatype':'comments',
-                    7:{'datatype':'bool', 'text':_('Yes/No'), 'is_multiple':False},
+                        'text':_('Long text, like comments, not shown in the tag browser'),
+                        'is_multiple':False},
+                    3:{'datatype':'datetime',
+                        'text':_('Date'), 'is_multiple':False},
+                    4:{'datatype':'float',
+                        'text':_('Floating point numbers'), 'is_multiple':False},
+                    5:{'datatype':'int',
+                        'text':_('Integers'), 'is_multiple':False},
+                    6:{'datatype':'rating',
+                        'text':_('Ratings, shown with stars'),
+                        'is_multiple':False},
+                    7:{'datatype':'bool',
+                        'text':_('Yes/No'), 'is_multiple':False},
                 }
+
     def __init__(self, parent, editing, standard_colheads, standard_colnames):
         QDialog.__init__(self, parent)
         Ui_QCreateCustomColumn.__init__(self)
         self.setupUi(self)
+        self.simple_error = partial(error_dialog, self, show=True,
+            show_copy_button=False)
         self.connect(self.button_box, SIGNAL("accepted()"), self.accept)
         self.connect(self.button_box, SIGNAL("rejected()"), self.reject)
         self.parent = parent
@@ -35,12 +55,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             return
         idx = parent.columns.currentRow()
         if idx < 0:
-            self.parent.messagebox(_('No column has been selected'))
+            return self.simple_error(_('No column selected'),
-            return
+                    _('No column has been selected'))
         col = unicode(parent.columns.item(idx).data(Qt.UserRole).toString())
         if col not in parent.custcols:
-            self.parent.messagebox(_('Selected column is not a user-defined column'))
+            return self.simple_error('', _('Selected column is not a user-defined column'))
-            return
 
         c = parent.custcols[col]
         self.column_name_box.setText(c['label'])
@@ -62,11 +81,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         else:
             is_multiple = False
         if not col:
-            self.parent.messagebox(_('No lookup name was provided'))
+            return self.simple_error('', _('No lookup name was provided'))
-            return
         if not col_heading:
-            self.parent.messagebox(_('No column heading was provided'))
+            return self.simple_error('', _('No column heading was provided'))
-            return
         bad_col = False
         if col in self.parent.custcols:
             if not self.editing_col or self.parent.custcols[col]['num'] != self.orig_column_number:
@@ -74,8 +91,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         if col in self.standard_colnames:
             bad_col = True
         if bad_col:
-            self.parent.messagebox(_('The lookup name %s is already used')%col)
+            return self.simple_error('', _('The lookup name %s is already used')%col)
-            return
         bad_head = False
         for t in self.parent.custcols:
             if self.parent.custcols[t]['name'] == col_heading:
@@ -85,11 +101,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             if self.standard_colheads[t] == col_heading:
                 bad_head = True
         if bad_head:
-            self.parent.messagebox(_('The heading %s is already used')%col_heading)
+            return self.simple_error('', _('The heading %s is already used')%col_heading)
-            return
         if ':' in col or ' ' in col or col.lower() != col:
-            self.parent.messagebox(_('The lookup name must be lower case and cannot contain ":"s or spaces'))
+            return self.simple_error('', _('The lookup name must be lower case and cannot contain ":"s or spaces'))
-            return
 
         if not self.editing_col:
             self.parent.custcols[col] = {

src/calibre/gui2/dialogs/config/create_custom_column.ui

@@ -9,8 +9,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>391</width>
+    <width>528</width>
-    <height>157</height>
+    <height>165</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -20,116 +20,119 @@
    </sizepolicy>
   </property>
   <property name="windowTitle">
-   <string>Create Tag-based Column</string>
+   <string>Create a custom column</string>
   </property>
-  <widget class="QWidget" name="verticalLayoutWidget">
+  <layout class="QVBoxLayout" name="verticalLayout">
-   <property name="geometry">
+   <item>
-    <rect>
+    <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0">
-     <x>10</x>
+     <property name="sizeConstraint">
-     <y>0</y>
+      <enum>QLayout::SetDefaultConstraint</enum>
-     <width>371</width>
+     </property>
-     <height>141</height>
+     <property name="margin">
-    </rect>
+      <number>5</number>
-   </property>
+     </property>
-   <layout class="QGridLayout" name="gridLayout_2" rowstretch="0,0,0,0">
+     <item row="2" column="0">
-    <property name="sizeConstraint">
+      <layout class="QGridLayout" name="gridLayout">
-     <enum>QLayout::SetDefaultConstraint</enum>
+       <item row="0" column="0">
-    </property>
+        <widget class="QLabel" name="label_2">
-    <property name="margin">
+         <property name="text">
-     <number>5</number>
+          <string>&amp;Lookup name</string>
-    </property>
+         </property>
-    <item row="2" column="0">
+         <property name="buddy">
-     <layout class="QGridLayout" name="gridLayout">
+          <cstring>column_name_box</cstring>
-      <item row="0" column="0">
+         </property>
-       <widget class="QLabel" name="label_2">
+        </widget>
-        <property name="text">
+       </item>
-         <string>Lookup name</string>
+       <item row="1" column="0">
-        </property>
+        <widget class="QLabel" name="label">
-       </widget>
+         <property name="text">
-      </item>
+          <string>Column &amp;heading</string>
-      <item row="1" column="0">
+         </property>
-       <widget class="QLabel" name="label">
+         <property name="buddy">
-        <property name="text">
+          <cstring>column_heading_box</cstring>
-         <string>Column heading</string>
+         </property>
-        </property>
+        </widget>
-       </widget>
+       </item>
-      </item>
+       <item row="0" column="1">
-      <item row="0" column="1">
+        <widget class="QLineEdit" name="column_name_box">
-       <widget class="QLineEdit" name="column_name_box">
+         <property name="minimumSize">
-        <property name="minimumSize">
+          <size>
-         <size>
+           <width>20</width>
-          <width>20</width>
+           <height>0</height>
-          <height>0</height>
+          </size>
-         </size>
+         </property>
-        </property>
+         <property name="toolTip">
-        <property name="toolTip">
+          <string>Used for searching the column. Must be lower case and not contain spaces or colons.</string>
-         <string>Used for searching the column. Must be lower case and not contain spaces or colons.</string>
+         </property>
-        </property>
+        </widget>
-       </widget>
+       </item>
-      </item>
+       <item row="1" column="1">
-      <item row="1" column="1">
+        <widget class="QLineEdit" name="column_heading_box">
-       <widget class="QLineEdit" name="column_heading_box">
+         <property name="toolTip">
-        <property name="toolTip">
+          <string>Column heading in the library view and category name in the tag browser</string>
-         <string>Column heading in the library view and category name in tags browser</string>
+         </property>
-        </property>
+        </widget>
-       </widget>
+       </item>
-      </item>
+       <item row="2" column="0">
-      <item row="2" column="0">
+        <widget class="QLabel" name="label_3">
-       <widget class="QLabel" name="label_3">
+         <property name="text">
-        <property name="text">
+          <string>Column &amp;type</string>
-         <string>Column type</string>
+         </property>
-        </property>
+         <property name="buddy">
-       </widget>
+          <cstring>column_type_box</cstring>
-      </item>
+         </property>
-      <item row="2" column="1">
+        </widget>
-       <widget class="QComboBox" name="column_type_box">
+       </item>
-        <property name="sizePolicy">
+       <item row="2" column="1">
-         <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
+        <widget class="QComboBox" name="column_type_box">
-          <horstretch>0</horstretch>
+         <property name="sizePolicy">
-          <verstretch>0</verstretch>
+          <sizepolicy hsizetype="Maximum" vsizetype="Fixed">
-         </sizepolicy>
+           <horstretch>0</horstretch>
-        </property>
+           <verstretch>0</verstretch>
-        <property name="minimumSize">
+          </sizepolicy>
-         <size>
+         </property>
-          <width>70</width>
+         <property name="minimumSize">
-          <height>0</height>
+          <size>
-         </size>
+           <width>70</width>
-        </property>
+           <height>0</height>
-        <property name="toolTip">
+          </size>
-         <string>What kind of information will be kept in the column.</string>
+         </property>
-        </property>
+         <property name="toolTip">
-       </widget>
+          <string>What kind of information will be kept in the column.</string>
-      </item>
+         </property>
-     </layout>
+        </widget>
-    </item>
+       </item>
-    <item row="3" column="0">
+      </layout>
-     <widget class="QDialogButtonBox" name="button_box">
+     </item>
-      <property name="orientation">
+     <item row="3" column="0">
-       <enum>Qt::Horizontal</enum>
+      <widget class="QDialogButtonBox" name="button_box">
-      </property>
+       <property name="orientation">
-      <property name="standardButtons">
+        <enum>Qt::Horizontal</enum>
-       <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
-      </property>
+       <property name="standardButtons">
-      <property name="centerButtons">
+        <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
-       <bool>true</bool>
+       </property>
-      </property>
+       <property name="centerButtons">
-     </widget>
+        <bool>true</bool>
-    </item>
+       </property>
-    <item row="1" column="0">
+      </widget>
-     <widget class="QLabel" name="label_6">
+     </item>
-      <property name="font">
+     <item row="1" column="0">
-       <font>
+      <widget class="QLabel" name="label_6">
-        <weight>75</weight>
+       <property name="font">
-        <bold>true</bold>
+        <font>
-       </font>
+         <weight>75</weight>
-      </property>
+         <bold>true</bold>
-      <property name="text">
+        </font>
-       <string>Create and edit custom columns</string>
+       </property>
-      </property>
+       <property name="text">
-     </widget>
+        <string>Create and edit custom columns</string>
-    </item>
+       </property>
-   </layout>
+      </widget>
-  </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
  </widget>
  <tabstops>
   <tabstop>column_name_box</tabstop>

src/calibre/gui2/dialogs/metadata_single.py

@@ -14,7 +14,7 @@ import traceback
 from PyQt4.Qt import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread, QDate, \
     QPixmap, QListWidgetItem, QDialog
 
-from calibre.gui2 import qstring_to_unicode, error_dialog, file_icon_provider, \
+from calibre.gui2 import error_dialog, file_icon_provider, \
                            choose_files, choose_images, ResizableDialog, \
                            warning_dialog
 from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog
@@ -552,12 +552,12 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def fetch_metadata(self):
         isbn   = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text()))
-        title  = qstring_to_unicode(self.title.text())
+        title  = unicode(self.title.text())
         try:
             author = string_to_authors(unicode(self.authors.text()))[0]
         except:
             author = ''
-        publisher = qstring_to_unicode(self.publisher.currentText())
+        publisher = unicode(self.publisher.currentText())
         if isbn or title or author or publisher:
             d = FetchMetadata(self, isbn, title, author, publisher, self.timeout)
             self._fetch_metadata_scope = d
@@ -623,12 +623,12 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def remove_unused_series(self):
         self.db.remove_unused_series()
-        idx = qstring_to_unicode(self.series.currentText())
+        idx = unicode(self.series.currentText())
         self.series.clear()
         self.initialize_series()
         if idx:
             for i in range(self.series.count()):
-                if qstring_to_unicode(self.series.itemText(i)) == idx:
+                if unicode(self.series.itemText(i)) == idx:
                     self.series.setCurrentIndex(i)
                     break
 
@@ -648,7 +648,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
             self.db.set_isbn(self.id,
                     re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())), notify=False)
             self.db.set_rating(self.id, 2*self.rating.value(), notify=False)
-            self.db.set_publisher(self.id, qstring_to_unicode(self.publisher.currentText()), notify=False)
+            self.db.set_publisher(self.id, unicode(self.publisher.currentText()), notify=False)
             self.db.set_tags(self.id, [x.strip() for x in
                 unicode(self.tags.text()).split(',')], notify=False)
             self.db.set_series(self.id,

src/calibre/gui2/dialogs/password.py

@@ -5,38 +5,38 @@ from PyQt4.QtGui import QDialog, QLineEdit
 from PyQt4.QtCore import SIGNAL, Qt
 
 from calibre.gui2.dialogs.password_ui import Ui_Dialog
-from calibre.gui2 import qstring_to_unicode, dynamic
+from calibre.gui2 import dynamic
 
 class PasswordDialog(QDialog, Ui_Dialog):
-    
+
     def __init__(self, window, name, msg):
         QDialog.__init__(self, window)
         Ui_Dialog.__init__(self)
         self.setupUi(self)
         self.cfg_key = re.sub(r'[^0-9a-zA-Z]', '_', name)
-        
+
         un = dynamic[self.cfg_key+'__un']
         pw = dynamic[self.cfg_key+'__pw']
         if not un: un = ''
         if not pw: pw = ''
         self.gui_username.setText(un)
         self.gui_password.setText(pw)
-        self.sname = name 
+        self.sname = name
         self.msg.setText(msg)
         self.connect(self.show_password, SIGNAL('stateChanged(int)'), self.toggle_password)
-        
+
     def toggle_password(self, state):
         if state == Qt.Unchecked:
             self.gui_password.setEchoMode(QLineEdit.Password)
         else:
             self.gui_password.setEchoMode(QLineEdit.Normal)
-    
+
     def username(self):
-        return qstring_to_unicode(self.gui_username.text())
+        return unicode(self.gui_username.text())
-    
+
     def password(self):
-        return qstring_to_unicode(self.gui_password.text())
+        return unicode(self.gui_password.text())
-    
+
     def accept(self):
         dynamic.set(self.cfg_key+'__un', unicode(self.gui_username.text()))
         dynamic.set(self.cfg_key+'__pw', unicode(self.gui_password.text()))

src/calibre/gui2/dialogs/scheduler.py

@@ -220,6 +220,10 @@ class Scheduler(QObject):
         self.cac = QAction(QIcon(I('user_profile.svg')), _('Add a custom news source'), self)
         self.connect(self.cac, SIGNAL('triggered(bool)'), self.customize_feeds)
         self.news_menu.addAction(self.cac)
+        self.news_menu.addSeparator()
+        self.all_action = self.news_menu.addAction(
+                _('Download all scheduled new sources'),
+                self.download_all_scheduled)
 
         self.timer = QTimer(self)
         self.timer.start(int(self.INTERVAL * 60000))
@@ -304,7 +308,11 @@ class Scheduler(QObject):
         if urn is not None:
             return self.download(urn)
         for urn in self.recipe_model.scheduled_urns():
-            self.download(urn)
+            if not self.download(urn):
+                break
+
+    def download_all_scheduled(self):
+        self.download_clicked(None)
 
     def download(self, urn):
         self.lock.lock()
@@ -316,12 +324,13 @@ class Scheduler(QObject):
                             'is active'))
                 d.setModal(False)
                 d.show()
-            return
+            return False
         self.internet_connection_failed = False
         doit = urn not in self.download_queue
         self.lock.unlock()
         if doit:
             self.do_download(urn)
+        return True
 
     def check(self):
         recipes = self.recipe_model.get_to_be_downloaded_recipes()

src/calibre/gui2/dialogs/search.py

@@ -4,7 +4,6 @@ import re
 from PyQt4.QtGui import QDialog
 
 from calibre.gui2.dialogs.search_ui import Ui_Dialog
-from calibre.gui2 import qstring_to_unicode
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
 
 class SearchDialog(QDialog, Ui_Dialog):
@@ -48,11 +47,11 @@ class SearchDialog(QDialog, Ui_Dialog):
         return ans
 
     def token(self):
-        txt = qstring_to_unicode(self.text.text()).strip()
+        txt = unicode(self.text.text()).strip()
         if txt:
             if self.negate.isChecked():
                 txt = '!'+txt
-            tok = self.FIELDS[qstring_to_unicode(self.field.currentText())]+txt
+            tok = self.FIELDS[unicode(self.field.currentText())]+txt
             if re.search(r'\s', tok):
                 tok = '"%s"'%tok
             return tok

src/calibre/gui2/dialogs/tag_categories.py

@@ -7,7 +7,7 @@ from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
-from calibre.gui2 import qstring_to_unicode, config
+from calibre.gui2 import config
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.constants import islinux
 
@@ -138,7 +138,7 @@ class TagCategories(QDialog, Ui_TagCategories):
 
     def add_category(self):
         self.save_category()
-        cat_name = qstring_to_unicode(self.input_box.text()).strip()
+        cat_name = unicode(self.input_box.text()).strip()
         if cat_name == '':
             return False
         if cat_name not in self.categories:

src/calibre/gui2/dialogs/tag_editor.py

@@ -4,7 +4,6 @@ from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog
 
 from calibre.gui2.dialogs.tag_editor_ui import Ui_TagEditor
-from calibre.gui2 import qstring_to_unicode
 from calibre.gui2 import question_dialog, error_dialog
 from calibre.constants import islinux
 
@@ -57,26 +56,26 @@ class TagEditor(QDialog, Ui_TagEditor):
             error_dialog(self, 'No tags selected', 'You must select at least one tag from the list of Available tags.').exec_()
             return
         for item in items:
-            if self.db.is_tag_used(qstring_to_unicode(item.text())):
+            if self.db.is_tag_used(unicode(item.text())):
                 confirms.append(item)
             else:
                 deletes.append(item)
         if confirms:
-            ct = ', '.join([qstring_to_unicode(item.text()) for item in confirms])
+            ct = ', '.join([unicode(item.text()) for item in confirms])
             if question_dialog(self, _('Are your sure?'),
                 '<p>'+_('The following tags are used by one or more books. '
                     'Are you certain you want to delete them?')+'<br>'+ct):
                 deletes += confirms
 
         for item in deletes:
-            self.db.delete_tag(qstring_to_unicode(item.text()))
+            self.db.delete_tag(unicode(item.text()))
             self.available_tags.takeItem(self.available_tags.row(item))
 
 
     def apply_tags(self, item=None):
         items = self.available_tags.selectedItems() if item is None else [item]
         for item in items:
-            tag = qstring_to_unicode(item.text())
+            tag = unicode(item.text())
             self.tags.append(tag)
             self.available_tags.takeItem(self.available_tags.row(item))
 
@@ -90,7 +89,7 @@ class TagEditor(QDialog, Ui_TagEditor):
     def unapply_tags(self, item=None):
         items = self.applied_tags.selectedItems() if item is None else [item]
         for item in items:
-            tag = qstring_to_unicode(item.text())
+            tag = unicode(item.text())
             self.tags.remove(tag)
             self.available_tags.addItem(tag)
 
@@ -102,7 +101,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         self.available_tags.sortItems()
 
     def add_tag(self):
-        tags = qstring_to_unicode(self.add_tag_input.text()).split(',')
+        tags = unicode(self.add_tag_input.text()).split(',')
         for tag in tags:
             tag = tag.strip()
             for item in self.available_tags.findItems(tag, Qt.MatchFixedString):

src/calibre/gui2/dialogs/user_profiles.py

@@ -9,7 +9,7 @@ from PyQt4.Qt import SIGNAL, QUrl, QDesktopServices, QAbstractListModel, Qt, \
 from calibre.web.feeds.recipes import compile_recipe
 from calibre.web.feeds.news import AutomaticNewsRecipe
 from calibre.gui2.dialogs.user_profiles_ui import Ui_Dialog
-from calibre.gui2 import qstring_to_unicode, error_dialog, question_dialog, \
+from calibre.gui2 import error_dialog, question_dialog, \
                          choose_files, ResizableDialog, NONE
 from calibre.gui2.widgets import PythonHighlighter
 from calibre.ptempfile import PersistentTemporaryFile
@@ -162,19 +162,19 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
         else:
             self.stacks.setCurrentIndex(1)
             self.toggle_mode_button.setText(_('Switch to Basic mode'))
-            if not qstring_to_unicode(self.source_code.toPlainText()).strip():
+            if not unicode(self.source_code.toPlainText()).strip():
                 src = self.options_to_profile()[0].replace('AutomaticNewsRecipe', 'BasicNewsRecipe')
                 self.source_code.setPlainText(src.replace('BasicUserRecipe', 'AdvancedUserRecipe'))
                 self.highlighter = PythonHighlighter(self.source_code.document())
 
 
     def add_feed(self, *args):
-        title = qstring_to_unicode(self.feed_title.text()).strip()
+        title = unicode(self.feed_title.text()).strip()
         if not title:
             error_dialog(self, _('Feed must have a title'),
                              _('The feed must have a title')).exec_()
             return
-        url = qstring_to_unicode(self.feed_url.text()).strip()
+        url = unicode(self.feed_url.text()).strip()
         if not url:
             error_dialog(self, _('Feed must have a URL'),
                          _('The feed %s must have a URL')%title).exec_()
@@ -190,7 +190,7 @@ class UserProfiles(ResizableDialog, Ui_Dialog):
 
     def options_to_profile(self):
         classname = 'BasicUserRecipe'+str(int(time.time()))
-        title = qstring_to_unicode(self.profile_title.text()).strip()
+        title = unicode(self.profile_title.text()).strip()
         if not title:
             title = classname
         self.profile_title.setText(title)
@@ -229,7 +229,7 @@ class %(classname)s(%(base_class)s):
                 return
             profile = src
         else:
-            src = qstring_to_unicode(self.source_code.toPlainText())
+            src = unicode(self.source_code.toPlainText())
             try:
                 title = compile_recipe(src).title
             except Exception, err:

src/calibre/gui2/library.py

@@ -19,7 +19,7 @@ from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, pyqtSignal, \
 from calibre import strftime
 from calibre.ebooks.metadata import string_to_authors, fmt_sidx, authors_to_string
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
-from calibre.gui2 import NONE, TableView, qstring_to_unicode, config, error_dialog
+from calibre.gui2 import NONE, TableView, config, error_dialog
 from calibre.gui2.dialogs.comments_dialog import CommentsDialog
 from calibre.gui2.widgets import EnLineEdit, TagsLineEdit
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
@@ -813,7 +813,7 @@ class BooksModel(QAbstractTableModel):
     def set_custom_column_data(self, row, colhead, value):
         typ = self.custom_columns[colhead]['datatype']
         if typ in ('text', 'comments'):
-            val = qstring_to_unicode(value.toString()).strip()
+            val = unicode(value.toString()).strip()
             val = val if val else None
         if typ == 'bool':
             val = value.toInt()[0] # tristate checkboxes put unknown in the middle
@@ -823,7 +823,7 @@ class BooksModel(QAbstractTableModel):
             val = 0 if val < 0 else 5 if val > 5 else val
             val *= 2
         elif typ in ('int', 'float'):
-            val = qstring_to_unicode(value.toString()).strip()
+            val = unicode(value.toString()).strip()
             if val is None or not val:
                 val = None
         elif typ == 'datetime':
@@ -1034,7 +1034,7 @@ class BooksView(TableView):
         and represent files with extensions.
         '''
         if event.mimeData().hasFormat('text/uri-list'):
-            urls = [qstring_to_unicode(u.toLocalFile()) for u in event.mimeData().urls()]
+            urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()]
             return [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)]
 
     def dragEnterEvent(self, event):
@@ -1390,7 +1390,7 @@ class DeviceBooksModel(BooksModel):
             row, col = index.row(), index.column()
             if col in [2, 3]:
                 return False
-            val = qstring_to_unicode(value.toString()).strip()
+            val = unicode(value.toString()).strip()
             idx = self.map[row]
             if col == 0:
                 self.db[idx].title = val

src/calibre/gui2/lrf_renderer/text.py

@@ -9,7 +9,6 @@ from PyQt4.QtGui import QFont, QColor, QPixmap, QGraphicsPixmapItem, \
 from calibre.ebooks.lrf.fonts import FONT_MAP
 from calibre.ebooks.BeautifulSoup import Tag
 from calibre.ebooks.hyphenate import hyphenate_word
-from calibre.gui2 import qstring_to_unicode
 
 WEIGHT_MAP = lambda wt : int((wt/10.)-1)
 NULL       = lambda a, b: a
@@ -527,12 +526,12 @@ class Line(QGraphicsItem):
             while True:
                 word = words.next()
                 word.highlight = False
-                if tokens[0] in qstring_to_unicode(word.string).lower():
+                if tokens[0] in unicode(word.string).lower():
                     matches.append(word)
                     for c in range(1, len(tokens)):
                         word = words.next()
                         print tokens[c], word.string
-                        if tokens[c] not in qstring_to_unicode(word.string):
+                        if tokens[c] not in unicode(word.string):
                             return None
                         matches.append(word)
                     for w in matches:
@@ -556,7 +555,7 @@ class Line(QGraphicsItem):
             if isinstance(tok, (int, float)):
                 s += ' '
             elif isinstance(tok, Word):
-                s += qstring_to_unicode(tok.string)
+                s += unicode(tok.string)
         return s
 
     def __str__(self):

src/calibre/gui2/status.py

@@ -7,7 +7,7 @@ from PyQt4.QtGui import QStatusBar, QLabel, QWidget, QHBoxLayout, QPixmap, \
 from PyQt4.QtCore import Qt, QSize, SIGNAL, QCoreApplication, pyqtSignal
 
 from calibre import fit_image, preferred_encoding, isosx
-from calibre.gui2 import qstring_to_unicode, config
+from calibre.gui2 import config
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.gui2.notify import get_notifier
@@ -260,7 +260,7 @@ class StatusBar(QStatusBar):
         return ret
 
     def jobs(self):
-        src = qstring_to_unicode(self.movie_button.jobs.text())
+        src = unicode(self.movie_button.jobs.text())
         return int(re.search(r'\d+', src).group())
 
     def show_book_info(self):
@@ -268,7 +268,7 @@ class StatusBar(QStatusBar):
 
     def job_added(self, nnum):
         jobs = self.movie_button.jobs
-        src = qstring_to_unicode(jobs.text())
+        src = unicode(jobs.text())
         num = self.jobs()
         text = src.replace(str(num), str(nnum))
         jobs.setText(text)
@@ -276,7 +276,7 @@ class StatusBar(QStatusBar):
 
     def job_done(self, nnum):
         jobs = self.movie_button.jobs
-        src = qstring_to_unicode(jobs.text())
+        src = unicode(jobs.text())
         num = self.jobs()
         text = src.replace(str(num), str(nnum))
         jobs.setText(text)

src/calibre/gui2/ui.py

@@ -893,7 +893,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 t = _("(all books)")
             else:
                 t = _("({0} of all)").format(self.current_view().row_count())
-            self.search_count.setStyleSheet('QLabel { background-color: white; }')
+            self.search_count.setStyleSheet(
+                    'QLabel { background-color: transparent; }')
         self.search_count.setText(t)
 
     def search_box_cleared(self):

src/calibre/gui2/viewer/bookmarkmanager.py

@@ -9,7 +9,7 @@ from PyQt4.Qt import Qt, QDialog, QAbstractTableModel, QVariant, SIGNAL, \
     QModelIndex, QInputDialog, QLineEdit, QFileDialog
 
 from calibre.gui2.viewer.bookmarkmanager_ui import Ui_BookmarkManager
-from calibre.gui2 import NONE, qstring_to_unicode
+from calibre.gui2 import NONE
 
 class BookmarkManager(QDialog, Ui_BookmarkManager):
     def __init__(self, parent, bookmarks):
@@ -111,7 +111,7 @@ class BookmarkTableModel(QAbstractTableModel):
 
     def setData(self, index, value, role):
         if role == Qt.EditRole:
-            self.bookmarks[index.row()] = (qstring_to_unicode(value.toString()).strip(), self.bookmarks[index.row()][1])
+            self.bookmarks[index.row()] = (unicode(value.toString()).strip(), self.bookmarks[index.row()][1])
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), index, index)
             return True
         return False

src/calibre/gui2/widgets.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import QListView, QIcon, QFont, QLabel, QListWidget, \
                         QMenu, QStringListModel, QCompleter, QStringList
 
 from calibre.gui2 import human_readable, NONE, TableView, \
-                         qstring_to_unicode, error_dialog, pixmap_to_data
+                         error_dialog, pixmap_to_data
 from calibre.gui2.dialogs.job_view_ui import Ui_Dialog
 from calibre.gui2.filename_pattern_ui import Ui_Form
 from calibre import fit_image
@@ -72,7 +72,7 @@ class FilenamePattern(QWidget, Ui_Form):
             error_dialog(self, _('Invalid regular expression'),
                          _('Invalid regular expression: %s')%err).exec_()
             return
-        mi = metadata_from_filename(qstring_to_unicode(self.filename.text()), pat)
+        mi = metadata_from_filename(unicode(self.filename.text()), pat)
         if mi.title:
             self.title.setText(mi.title)
         else:
@@ -96,7 +96,7 @@ class FilenamePattern(QWidget, Ui_Form):
 
 
     def pattern(self):
-        pat = qstring_to_unicode(self.re.text())
+        pat = unicode(self.re.text())
         return re.compile(pat)
 
     def commit(self):
@@ -158,7 +158,7 @@ class ImageView(QLabel):
         and represent files with extensions.
         '''
         if event.mimeData().hasFormat('text/uri-list'):
-            urls = [qstring_to_unicode(u.toLocalFile()) for u in event.mimeData().urls()]
+            urls = [unicode(u.toLocalFile()) for u in event.mimeData().urls()]
             urls = [u for u in urls if os.path.splitext(u)[1] and os.access(u, os.R_OK)]
             return [u for u in urls if os.path.splitext(u)[1][1:].lower() in cls.DROPABBLE_EXTENSIONS]
 
@@ -630,13 +630,13 @@ class TagsLineEdit(EnLineEdit):
         self.completer.update_tags_cache(tags)
 
     def text_changed(self, text):
-        all_text = qstring_to_unicode(text)
+        all_text = unicode(text)
         text = all_text[:self.cursorPosition()]
         prefix = text.split(',')[-1].strip()
 
         text_tags = []
         for t in all_text.split(self.separator):
-            t1 = qstring_to_unicode(t).strip()
+            t1 = unicode(t).strip()
             if t1 != '':
                 text_tags.append(t)
         text_tags = list(set(text_tags))
@@ -646,8 +646,8 @@ class TagsLineEdit(EnLineEdit):
 
     def complete_text(self, text):
         cursor_pos = self.cursorPosition()
-        before_text = qstring_to_unicode(self.text())[:cursor_pos]
+        before_text = unicode(self.text())[:cursor_pos]
-        after_text = qstring_to_unicode(self.text())[cursor_pos:]
+        after_text = unicode(self.text())[cursor_pos:]
         prefix_len = len(before_text.split(',')[-1].strip())
         self.setText('%s%s%s %s' % (before_text[:cursor_pos - prefix_len],
             text, self.separator, after_text))

src/calibre/library/caches.py

@@ -526,7 +526,7 @@ class ResultCache(SearchQueryParser):
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]
 
-    def search(self, query, return_matches = False):
+    def search(self, query, return_matches=False):
         if not query or not query.strip():
             q = self.search_restriction
         else:

src/calibre/library/custom_columns.py

@@ -45,6 +45,7 @@ class CustomColumns(object):
                     DROP TRIGGER IF EXISTS fkc_insert_{table};
                     DROP TRIGGER IF EXISTS fkc_delete_{table};
                     DROP VIEW    IF EXISTS tag_browser_{table};
+                    DROP VIEW    IF EXISTS tag_browser_filtered_{table};
                     DROP TABLE   IF EXISTS {table};
                     DROP TABLE   IF EXISTS {lt};
                     '''.format(table=table, lt=lt)
@@ -137,7 +138,14 @@ class CustomColumns(object):
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
                 'datetime' : adapt_datetime,
                 'text':adapt_text
-                }
+        }
+
+        # Create Tag Browser categories for custom columns
+        for i, v in self.custom_column_num_map.items():
+            if v['normalized']:
+                tn = 'custom_column_{0}'.format(i)
+                self.tag_browser_categories[tn] = [v['label'], 'value']
+
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:
@@ -396,6 +404,13 @@ class CustomColumns(object):
                     (SELECT COUNT(id) FROM {lt} WHERE value={table}.id) count
                 FROM {table};
 
+                CREATE VIEW tag_browser_filtered_{table} AS SELECT
+                    id,
+                    value,
+                    (SELECT COUNT({lt}.id) FROM {lt} WHERE value={table}.id AND
+                    books_list_filter(book)) count
+                FROM {table};
+
                 '''.format(lt=lt, table=table),
 
             ]

src/calibre/library/database2.py

@@ -106,6 +106,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.conn = connect(self.dbpath, self.row_factory)
         if self.user_version == 0:
             self.initialize_database()
+        self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
 
     def __init__(self, library_path, row_factory=False):
         if not os.path.exists(library_path):
@@ -118,6 +119,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 self.dbpath)
         if isinstance(self.dbpath, unicode):
             self.dbpath = self.dbpath.encode(filesystem_encoding)
+
+        self.tag_browser_categories = {
+                'tags'      : ['tag', 'name'],
+                'series'    : ['series', 'name'],
+                'publishers': ['publisher', 'name'],
+                'authors'   : ['author', 'name'],
+                'news'      : ['news', 'name'],
+        }
+
         self.connect()
         self.is_case_sensitive = not iswindows and not isosx and \
             not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
@@ -125,6 +135,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.initialize_dynamic()
 
     def initialize_dynamic(self):
+        self.conn.executescript(u'''
+            CREATE TEMP VIEW IF NOT EXISTS tag_browser_news AS SELECT DISTINCT
+                id,
+                name,
+                (SELECT COUNT(books_tags_link.id) FROM books_tags_link WHERE tag=x.id) count
+            FROM tags as x WHERE name!="{0}" AND id IN
+                (SELECT DISTINCT tag FROM books_tags_link WHERE book IN
+                    (SELECT DISTINCT book FROM books_tags_link WHERE tag IN
+                        (SELECT id FROM tags WHERE name="{0}")));
+            '''.format(_('News')))
+
+        self.conn.executescript(u'''
+            CREATE TEMP VIEW IF NOT EXISTS tag_browser_filtered_news AS SELECT DISTINCT
+                id,
+                name,
+                (SELECT COUNT(books_tags_link.id) FROM books_tags_link WHERE tag=x.id and books_list_filter(book)) count
+            FROM tags as x WHERE name!="{0}" AND id IN
+                (SELECT DISTINCT tag FROM books_tags_link WHERE book IN
+                    (SELECT DISTINCT book FROM books_tags_link WHERE tag IN
+                        (SELECT id FROM tags WHERE name="{0}")));
+            '''.format(_('News')))
+        self.conn.commit()
+
+
         CustomColumns.__init__(self)
         template = '''\
                 (SELECT {query} FROM books_{table}_link AS link INNER JOIN
@@ -576,68 +610,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
 
     def get_categories(self, sort_on_count=False, ids=None, icon_map=None):
-
+        self.books_list_filter.change([] if not ids else ids)
-        orig_category_columns = {'tags': ['tag', 'name'],
-                                 'series': ['series', 'name'],
-                                 'publishers': ['publisher', 'name'],
-                                 'authors': ['author', 'name']} # 'news' is added below
-        cat_cols = {}
-
-        def create_filtered_views(self, ids):
-            def create_tag_browser_view(table_name, column_name, view_column_name):
-                script = ('''
-                    CREATE TEMP VIEW IF NOT EXISTS tag_browser_filtered_{tn} AS SELECT
-                        id,
-                        {vcn},
-                        (SELECT COUNT(books_{tn}_link.id) FROM books_{tn}_link WHERE {cn}={tn}.id and books_list_filter(book)) count
-                    FROM {tn};
-                    '''.format(tn=table_name, cn=column_name, vcn=view_column_name))
-                self.conn.executescript(script)
-
-            self.cat_cols = {}
-            for tn,cn in orig_category_columns.iteritems():
-                create_tag_browser_view(tn, cn[0], cn[1])
-                cat_cols[tn] = cn
-            for i,v in self.custom_column_num_map.iteritems():
-                if v['datatype'] == 'text':
-                    tn = 'custom_column_{0}'.format(i)
-                    create_tag_browser_view(tn, 'value', 'value')
-                    cat_cols[tn] = [v['label'], 'value']
-            cat_cols['news'] = ['news', 'name']
-
-            self.conn.executescript(u'''
-                CREATE TEMP VIEW IF NOT EXISTS tag_browser_news AS SELECT DISTINCT
-                    id,
-                    name,
-                    (SELECT COUNT(books_tags_link.id) FROM books_tags_link WHERE tag=x.id) count
-                FROM tags as x WHERE name!="{0}" AND id IN
-                    (SELECT DISTINCT tag FROM books_tags_link WHERE book IN
-                        (SELECT DISTINCT book FROM books_tags_link WHERE tag IN
-                            (SELECT id FROM tags WHERE name="{0}")));
-                '''.format(_('News')))
-            self.conn.commit()
-
-            self.conn.executescript(u'''
-                CREATE TEMP VIEW IF NOT EXISTS tag_browser_filtered_news AS SELECT DISTINCT
-                    id,
-                    name,
-                    (SELECT COUNT(books_tags_link.id) FROM books_tags_link WHERE tag=x.id and books_list_filter(book)) count
-                FROM tags as x WHERE name!="{0}" AND id IN
-                    (SELECT DISTINCT tag FROM books_tags_link WHERE book IN
-                        (SELECT DISTINCT book FROM books_tags_link WHERE tag IN
-                            (SELECT id FROM tags WHERE name="{0}")));
-                '''.format(_('News')))
-            self.conn.commit()
-
-        if ids is not None:
-            s_ids = set(ids)
-        else:
-            s_ids = None
-        self.conn.create_function('books_list_filter', 1, lambda(id): 1 if id in s_ids else 0)
-        create_filtered_views(self, ids)
 
         categories = {}
-        for tn,cn in cat_cols.iteritems():
+        for tn, cn in self.tag_browser_categories.items():
             if ids is None:
                 query = 'SELECT id, {0}, count FROM tag_browser_{1}'.format(cn[1], tn)
             else:
@@ -648,12 +624,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 query += ' ORDER BY {0} ASC'.format(cn[1])
             data = self.conn.get(query)
             category = cn[0]
-            if category in icon_map:
+            icon, tooltip = None, ''
-                icon = icon_map[category]
+            if icon_map:
-                tooltip = ''
+                if category in icon_map:
-            else:
+                    icon = icon_map[category]
-                icon = icon_map['*custom']
+                    tooltip = ''
-                tooltip = self.custom_column_label_map[category]['name']
+                else:
+                    icon = icon_map['*custom']
+                    tooltip = self.custom_column_label_map[category]['name']
             if ids is None: # no filtering
                 categories[category] = [Tag(r[1], count=r[2], id=r[0], icon=icon, tooltip = tooltip)
                                         for r in data]
@@ -666,14 +644,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if ids is not None:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
-                                       WHERE format="%s" and books_list_filter(id)'''%fmt,
+                                       WHERE format="%s" AND books_list_filter(id)'''%fmt,
                                        all=False)
             else:
                 count = self.conn.get('''SELECT COUNT(id)
                                        FROM data
                                        WHERE format="%s"'''%fmt,
                                        all=False)
-            categories['format'].append(Tag(fmt, count=count))
+            if count > 0:
+                categories['format'].append(Tag(fmt, count=count))
 
         if sort_on_count:
             categories['format'].sort(cmp=lambda x,y:cmp(x.count, y.count),
@@ -1475,6 +1454,7 @@ books_series_link      feeds
             conn = ndb.conn
             conn.execute('create table temp_sequence(id INTEGER PRIMARY KEY AUTOINCREMENT)')
             conn.commit()
+            conn.create_function(self.books_list_filter.name, 1, lambda x: 1)
             conn.executescript(sql)
             conn.commit()
             conn.execute('pragma user_version=%d'%user_version)

src/calibre/library/schema_upgrades.py

@@ -269,3 +269,22 @@ class SchemaUpgrade(object):
                 CREATE INDEX IF NOT EXISTS formats_idx ON data (format);
         ''')
 
+    def upgrade_version_10(self):
+        'Add restricted Tag Browser views'
+        def create_tag_browser_view(table_name, column_name, view_column_name):
+            script = ('''
+                DROP VIEW IF EXISTS tag_browser_filtered_{tn};
+                CREATE VIEW tag_browser_filtered_{tn} AS SELECT
+                    id,
+                    {vcn},
+                    (SELECT COUNT(books_{tn}_link.id) FROM books_{tn}_link WHERE
+                        {cn}={tn}.id AND books_list_filter(book)) count
+                FROM {tn};
+                '''.format(tn=table_name, cn=column_name, vcn=view_column_name))
+            self.conn.executescript(script)
+
+        for tn, cn in self.tag_browser_categories.items():
+            if tn != 'news':
+                create_tag_browser_view(tn, cn[0], cn[1])
+
+

src/calibre/library/sqlite.py

@@ -36,6 +36,18 @@ def convert_bool(val):
 sqlite.register_adapter(bool, lambda x : 1 if x else 0)
 sqlite.register_converter('bool', convert_bool)
 
+class DynamicFilter(object):
+
+    def __init__(self, name):
+        self.name = name
+        self.ids = frozenset([])
+
+    def __call__(self, id_):
+        return int(id_ in self.ids)
+
+    def change(self, ids):
+        self.ids = frozenset(ids)
+
 
 class Concatenate(object):
     '''String concatenation aggregator for sqlite'''
@@ -119,6 +131,13 @@ class DBThread(Thread):
                         ok, res = True, '\n'.join(self.conn.iterdump())
                     except Exception, err:
                         ok, res = False, (err, traceback.format_exc())
+                elif func == 'create_dynamic_filter':
+                    try:
+                        f = DynamicFilter(args[0])
+                        self.conn.create_function(args[0], 1, f)
+                        ok, res = True, f
+                    except Exception, err:
+                        ok, res = False, (err, traceback.format_exc())
                 else:
                     func = getattr(self.conn, func)
                     try:
@@ -203,6 +222,9 @@ class ConnectionProxy(object):
     @proxy
     def dump(self): pass
 
+    @proxy
+    def create_dynamic_filter(self): pass
+
 def connect(dbpath, row_factory=None):
     conn = ConnectionProxy(DBThread(dbpath, row_factory))
     conn.proxy.start()

src/calibre/manual/plugins.rst

@@ -113,7 +113,7 @@ Metadata download plugins
     When :meth:`fetch` is called, the `self` object will have the following
     useful attributes (each of which may be None)::
 
-        title, author, publisher, isbn, log, verbose and extra
+        title, book_author, publisher, isbn, log, verbose and extra
 
     Use these attributes to construct the search query. extra is reserved for
     future use.