From 8b03a506a3e9ff7ef1864fe51efde215f32db032 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 3 Jan 2011 15:04:36 -0700
Subject: [PATCH 01/33] ...

---
 setup/publish.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/publish.py b/setup/publish.py
index fb873a90d9..ba8a4992a7 100644
--- a/setup/publish.py
+++ b/setup/publish.py
@@ -43,8 +43,8 @@ class Stage3(Command):
 
    description = 'Stage 3 of the publish process'
    sub_commands = ['upload_user_manual', 'upload_demo', 'sdist',
-           'upload_to_mobileread', 'upload_to_google_code',
-           'tag_release', 'upload_to_server', 'upload_to_sourceforge',
+            'upload_to_google_code', 'tag_release', 'upload_to_server',
+            'upload_to_sourceforge', 'upload_to_mobileread',
    ]
 
 class Stage4(Command):

From dd6b0c7efc5bfd6b3f94857180a25896502366a8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 3 Jan 2011 15:13:05 -0700
Subject: [PATCH 02/33] ImageMagick interface: Don't crash when asked to open
 empty image files

---
 src/calibre/utils/magick/__init__.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/calibre/utils/magick/__init__.py b/src/calibre/utils/magick/__init__.py
index bf0f48db7d..834a798de5 100644
--- a/src/calibre/utils/magick/__init__.py
+++ b/src/calibre/utils/magick/__init__.py
@@ -106,13 +106,16 @@ class Image(_magick.Image): # {{{
         return ans
 
     def load(self, data):
-        return _magick.Image.load(self, bytes(data))
+        data = bytes(data)
+        if not data:
+            raise ValueError('Cannot open image from empty data string')
+        return _magick.Image.load(self, data)
 
     def open(self, path_or_file):
         if not hasattr(path_or_file, 'read') and \
             path_or_file.lower().endswith('.wmf'):
             # Special handling for WMF files as ImageMagick seems
-            # to hand while reading them from a blob on linux
+            # to hang while reading them from a blob on linux
             if isinstance(path_or_file, unicode):
                 path_or_file = path_or_file.encode(filesystem_encoding)
             return _magick.Image.read(self, path_or_file)
@@ -121,6 +124,8 @@ class Image(_magick.Image): # {{{
             data = data.read()
         else:
             data = open(data, 'rb').read()
+        if not data:
+            raise ValueError('%r is an empty file'%path_or_file)
         self.load(data)
 
     @dynamic_property

From c483d01b45f25bd1326052c2e352b51cca6613f1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 3 Jan 2011 17:27:37 -0700
Subject: [PATCH 03/33] Fix bug where composite custom columns using
 general_program_mode fields are not evaluated correctly when used in a
 template.

---
 resources/default_tweaks.py          |  9 ++++++---
 src/calibre/manual/template_lang.rst |  4 ++--
 src/calibre/utils/formatter.py       | 10 +++++-----
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py
index 4ae0278133..32aeba9122 100644
--- a/resources/default_tweaks.py
+++ b/resources/default_tweaks.py
@@ -69,9 +69,12 @@ categories_use_field_for_author_name = 'author'
 #  avg_rating: the averate rating of all the books referencing this item
 #  sort: the sort value. For authors, this is the author_sort for that author
 #  category: the category (e.g., authors, series) that the item is in.
-categories_collapsed_name_template = '{first.sort:shorten(4,'',0)} - {last.sort:shorten(4,'',0)}'
-categories_collapsed_rating_template = '{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
-categories_collapsed_popularity_template = '{first.count:d} - {last.count:d}'
+# Note that the "r'" in front of the { is necessary if there are backslashes
+# (\ characters) in the template. It doesn't hurt anything to leave it there
+# even if there aren't any backslashes.
+categories_collapsed_name_template = r'{first.sort:shorten(4,'',0)} - {last.sort:shorten(4,'',0)}'
+categories_collapsed_rating_template = r'{first.avg_rating:4.2f:ifempty(0)} - {last.avg_rating:4.2f:ifempty(0)}'
+categories_collapsed_popularity_template = r'{first.count:d} - {last.count:d}'
 
 
 # Set whether boolean custom columns are two- or three-valued.
diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst
index 8a3bd854b1..1bf08c11f9 100644
--- a/src/calibre/manual/template_lang.rst
+++ b/src/calibre/manual/template_lang.rst
@@ -150,7 +150,7 @@ The example shows several things:
 
     * program mode is used if the expression begins with ``:'`` and ends with ``'``. Anything else is assumed to be single-function.
     * the variable ``$`` stands for the field the expression is operating upon, ``#series`` in this case.
-    * functions must be given all their arguments. There is no default value. This is true for the standard builtin functions, and is a significant difference from single-function mode.
+    * functions must be given all their arguments. There is no default value. For example, the standard builtin functions must be given an additional initial parameter indicating the source field, which is a significant difference from single-function mode.
     * white space is ignored and can be used anywhere within the expression.
     * constant strings are enclosed in matching quotes, either ``'`` or ``"``.
     
@@ -204,7 +204,7 @@ For various values of series_index, the program returns:
 
 All the functions listed under single-function mode can be used in program mode, noting that unlike the functions described below you must supply a first parameter providing the value the function is to act upon. 
 
-The following functions are available in addition to those described in single-function mode. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
+The following functions are available in addition to those described in single-function mode. Remember from the example above that the single-function mode functions require an additional first parameter specifying the field to operate on. With the exception of the ``id`` parameter of assign, all parameters can be statements (sequences of expressions):
 
     * ``add(x, y)`` -- returns x + y. Throws an exception if either x or y are not numbers.
     * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index 4fe8ad2e4f..4e9710ad14 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -431,7 +431,10 @@ class TemplateFormatter(string.Formatter):
         return prefix + val + suffix
 
     def vformat(self, fmt, args, kwargs):
-        ans = string.Formatter.vformat(self, fmt, args, kwargs)
+        if fmt.startswith('program:'):
+            ans = self._eval_program(None, fmt[8:])
+        else:
+            ans = string.Formatter.vformat(self, fmt, args, kwargs)
         return self.compress_spaces.sub(' ', ans).strip()
 
     ########## a formatter guaranteed not to throw and exception ############
@@ -441,10 +444,7 @@ class TemplateFormatter(string.Formatter):
         self.book = book
         self.composite_values = {}
         try:
-            if fmt.startswith('program:'):
-                ans = self._eval_program(None, fmt[8:])
-            else:
-                ans = self.vformat(fmt, [], kwargs).strip()
+            ans = self.vformat(fmt, [], kwargs).strip()
         except Exception, e:
             if DEBUG:
                 traceback.print_exc()

From 19db50ae44db6d7fef85f09eef3a411bd458bbf2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 3 Jan 2011 18:17:21 -0700
Subject: [PATCH 04/33] Fix #8161 (Ability to send updated thumbnails to newer
 Sony readers)

---
 src/calibre/devices/prs505/driver.py          | 28 +++++++++---
 src/calibre/devices/usbms/deviceconfig.py     | 38 ++++++++++++++--
 .../gui2/device_drivers/configwidget.py       | 43 ++++++++++++++++---
 .../gui2/device_drivers/configwidget.ui       | 15 +------
 src/calibre/library/catalog.py                |  2 +-
 5 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index 6652d581d4..98a7241a36 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -61,14 +61,26 @@ class PRS505(USBMS):
     ALL_BY_TITLE  = _('All by title')
     ALL_BY_AUTHOR = _('All by author')
 
-    EXTRA_CUSTOMIZATION_MESSAGE = _('Comma separated list of metadata fields '
+    EXTRA_CUSTOMIZATION_MESSAGE = [
+        _('Comma separated list of metadata fields '
             'to turn into collections on the device. Possibilities include: ')+\
                     'series, tags, authors' +\
             _('. Two special collections are available: %s:%s and %s:%s. Add  '
             'these values to the list to enable them. The collections will be '
             'given the name provided after the ":" character.')%(
-                                    'abt', ALL_BY_TITLE, 'aba', ALL_BY_AUTHOR)
-    EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['series', 'tags'])
+                                    'abt', ALL_BY_TITLE, 'aba', ALL_BY_AUTHOR),
+            _('Upload separate cover thumbnails for books (newer readers)') +
+            ':::'+_('Normally, the SONY readers get the cover image from the'
+                ' ebook file itself. With this option, calibre will send a '
+                'separate cover image to the reader, useful if you are '
+                'sending DRMed books in which you cannot change the cover.'
+                ' WARNING: This option should only be used with newer '
+                'SONY readers: 350, 650, 950 and newer.'),
+    ]
+    EXTRA_CUSTOMIZATION_DEFAULT = [
+                ', '.join(['series', 'tags']),
+                False
+    ]
 
     plugboard = None
     plugboard_func = None
@@ -159,7 +171,7 @@ class PRS505(USBMS):
         opts = self.settings()
         if opts.extra_customization:
             collections = [x.strip() for x in
-                    opts.extra_customization.split(',')]
+                    opts.extra_customization[0].split(',')]
         else:
             collections = []
         debug_print('PRS505: collection fields:', collections)
@@ -186,8 +198,12 @@ class PRS505(USBMS):
         self.plugboard_func = pb_func
 
     def upload_cover(self, path, filename, metadata, filepath):
-        return # Disabled as the SONY's don't need this thumbnail anyway and
-               # older models don't auto delete it
+        opts = self.settings()
+        if not opts.extra_customization[1]:
+            # Building thumbnails disabled
+            debug_print('PRS505: not uploading covers')
+            return
+        debug_print('PRS505: uploading covers')
         if metadata.thumbnail and metadata.thumbnail[-1]:
             path = path.replace('/', os.sep)
             is_main = path.startswith(self._main_prefix)
diff --git a/src/calibre/devices/usbms/deviceconfig.py b/src/calibre/devices/usbms/deviceconfig.py
index e074387175..769543f7cc 100644
--- a/src/calibre/devices/usbms/deviceconfig.py
+++ b/src/calibre/devices/usbms/deviceconfig.py
@@ -10,7 +10,21 @@ from calibre.utils.config import Config, ConfigProxy
 class DeviceConfig(object):
 
     HELP_MESSAGE = _('Configure Device')
+
+    #: Can be None, a string or a list of strings. When it is a string
+    #: that string is used for the help text and the actual customization value
+    #: can be read from ``dev.settings().extra_customization``.
+    #: If it a list of strings, then dev.settings().extra_customization will
+    #: also be a list. In this case, you *must* ensure that
+    #: EXTRA_CUSTOMIZATION_DEFAULT is also a list. The list can contain either
+    #: boolean values or strings, in which case a checkbox or line edit will be
+    #: used for them in the config widget, automatically.
+    #: If a string contains ::: then the text after it is interpreted as the
+    #: tooltip
     EXTRA_CUSTOMIZATION_MESSAGE = None
+
+    #: The default value for extra customization. If you set
+    #: EXTRA_CUSTOMIZATION_MESSAGE you *must* set this as well.
     EXTRA_CUSTOMIZATION_DEFAULT = None
 
     SUPPORTS_SUB_DIRS = False
@@ -73,16 +87,32 @@ class DeviceConfig(object):
         if cls.SUPPORTS_USE_AUTHOR_SORT:
             proxy['use_author_sort'] = config_widget.use_author_sort()
         if cls.EXTRA_CUSTOMIZATION_MESSAGE:
-            ec = unicode(config_widget.opt_extra_customization.text()).strip()
-            if not ec:
-                ec = None
+            if isinstance(cls.EXTRA_CUSTOMIZATION_MESSAGE, list):
+                ec = []
+                for i in range(0, len(cls.EXTRA_CUSTOMIZATION_MESSAGE)):
+                    if hasattr(config_widget.opt_extra_customization[i], 'isChecked'):
+                        ec.append(config_widget.opt_extra_customization[i].isChecked())
+                    else:
+                        ec.append(unicode(config_widget.opt_extra_customization[i].text()).strip())
+            else:
+                ec = unicode(config_widget.opt_extra_customization.text()).strip()
+                if not ec:
+                    ec = None
             proxy['extra_customization'] = ec
         st = unicode(config_widget.opt_save_template.text())
         proxy['save_template'] = st
 
     @classmethod
     def settings(cls):
-        return cls._config().parse()
+        opts = cls._config().parse()
+        if isinstance(cls.EXTRA_CUSTOMIZATION_DEFAULT, list):
+            if not isinstance(opts.extra_customization, list):
+                opts.extra_customization = [opts.extra_customization]
+            else:
+                for i,d in enumerate(cls.EXTRA_CUSTOMIZATION_DEFAULT):
+                    if i >= len(opts.extra_customization):
+                        opts.extra_customization.append(d)
+        return opts
 
     @classmethod
     def save_template(cls):
diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py
index 64321e1a46..7b440db7fc 100644
--- a/src/calibre/gui2/device_drivers/configwidget.py
+++ b/src/calibre/gui2/device_drivers/configwidget.py
@@ -4,7 +4,10 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL
+import textwrap
+
+from PyQt4.Qt import QWidget, QListWidgetItem, Qt, QVariant, SIGNAL, \
+                     QLabel, QLineEdit, QCheckBox
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.device_drivers.configwidget_ui import Ui_ConfigWidget
@@ -46,12 +49,38 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         else:
             self.opt_use_author_sort.hide()
         if extra_customization_message:
-            self.extra_customization_label.setText(extra_customization_message)
-            if settings.extra_customization:
-                self.opt_extra_customization.setText(settings.extra_customization)
-        else:
-            self.extra_customization_label.setVisible(False)
-            self.opt_extra_customization.setVisible(False)
+            def parse_msg(m):
+                msg, _, tt = m.partition(':::') if m else ('', '', '')
+                return msg.strip(), textwrap.fill(tt.strip(), 100)
+
+            if isinstance(extra_customization_message, list):
+                self.opt_extra_customization = []
+                for i, m in enumerate(extra_customization_message):
+                    label_text, tt = parse_msg(m)
+                    if isinstance(settings.extra_customization[i], bool):
+                        self.opt_extra_customization.append(QCheckBox(label_text))
+                        self.opt_extra_customization[-1].setToolTip(tt)
+                        self.opt_extra_customization[i].setChecked(bool(settings.extra_customization[i]))
+                    else:
+                        self.opt_extra_customization.append(QLineEdit(self))
+                        l = QLabel(label_text)
+                        l.setToolTip(tt)
+                        l.setBuddy(self.opt_extra_customization[i])
+                        l.setWordWrap(True)
+                        self.opt_extra_customization[i].setText(settings.extra_customization[i])
+                        self.extra_layout.addWidget(l)
+                    self.extra_layout.addWidget(self.opt_extra_customization[i])
+            else:
+                self.opt_extra_customization = QLineEdit()
+                label_text, tt = parse_msg(extra_customization_message)
+                l = QLabel(label_text)
+                l.setToolTip(tt)
+                l.setBuddy(self.opt_extra_customization)
+                l.setWordWrap(True)
+                if settings.extra_customization:
+                    self.opt_extra_customization.setText(settings.extra_customization)
+                self.extra_layout.addWidget(l)
+                self.extra_layout.addWidget(self.opt_extra_customization)
         self.opt_save_template.setText(settings.save_template)
 
 
diff --git a/src/calibre/gui2/device_drivers/configwidget.ui b/src/calibre/gui2/device_drivers/configwidget.ui
index 1e8ee75852..f4902a7387 100644
--- a/src/calibre/gui2/device_drivers/configwidget.ui
+++ b/src/calibre/gui2/device_drivers/configwidget.ui
@@ -98,20 +98,7 @@
     </widget>
    </item>
    <item row="6" column="0">
-    <widget class="QLabel" name="extra_customization_label">
-     <property name="text">
-      <string>Extra customization</string>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-     <property name="buddy">
-      <cstring>opt_extra_customization</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="7" column="0">
-    <widget class="QLineEdit" name="opt_extra_customization"/>
+    <layout class="QVBoxLayout" name="extra_layout"/>
    </item>
    <item row="4" column="0">
     <widget class="QLabel" name="label">
diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py
index ae61d7cf52..0a5d5284e2 100644
--- a/src/calibre/library/catalog.py
+++ b/src/calibre/library/catalog.py
@@ -15,7 +15,7 @@ from calibre.customize import CatalogPlugin
 from calibre.customize.conversion import OptionRecommendation, DummyReporter
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString
 from calibre.ebooks.chardet import substitute_entites
-from calibre.ebooks.oeb.base import RECOVER_PARSER, XHTML_NS
+from calibre.ebooks.oeb.base import XHTML_NS
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.config import config_dir
 from calibre.utils.date import format_date, isoformat, now as nowf

From 19562957ae02a5e38d37c6815f3c7e38eb059153 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 3 Jan 2011 18:20:55 -0700
Subject: [PATCH 05/33] Fix #8175 (Double-click on plugin in Preferences dialog
 to open customize dialog)

---
 src/calibre/gui2/preferences/plugins.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index b18159cce5..3d714e388e 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -129,6 +129,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.plugin_view.setModel(self._plugin_model)
         self.plugin_view.setStyleSheet(
                 "QTreeView::item { padding-bottom: 10px;}")
+        self.plugin_view.doubleClicked.connect(self.double_clicked)
         self.toggle_plugin_button.clicked.connect(self.toggle_plugin)
         self.customize_plugin_button.clicked.connect(self.customize_plugin)
         self.remove_plugin_button.clicked.connect(self.remove_plugin)
@@ -138,6 +139,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
     def toggle_plugin(self, *args):
         self.modify_plugin(op='toggle')
 
+    def double_clicked(self, index):
+        if index.parent().isValid():
+            self.modify_plugin(op='customize')
+
     def customize_plugin(self, *args):
         self.modify_plugin(op='customize')
 

From f77d55f06ae43ad1ee3348b014881fea8438f5e4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 08:39:37 -0700
Subject: [PATCH 06/33] Update ImageMagick in windows build to 6.6.6

---
 setup/build_environment.py        | 2 +-
 setup/installer/windows/freeze.py | 2 +-
 setup/installer/windows/notes.rst | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/setup/build_environment.py b/setup/build_environment.py
index d6581a907d..10ab1b0735 100644
--- a/setup/build_environment.py
+++ b/setup/build_environment.py
@@ -121,7 +121,7 @@ if iswindows:
     poppler_lib_dirs = consolidate('POPPLER_LIB_DIR', sw_lib_dir)
     popplerqt4_lib_dirs = poppler_lib_dirs
     poppler_libs = ['poppler']
-    magick_inc_dirs = [os.path.join(prefix, 'build', 'ImageMagick-6.5.6')]
+    magick_inc_dirs = [os.path.join(prefix, 'build', 'ImageMagick-6.6.6')]
     magick_lib_dirs = [os.path.join(magick_inc_dirs[0], 'VisualMagick', 'lib')]
     magick_libs = ['CORE_RL_wand_', 'CORE_RL_magick_']
     podofo_inc = os.path.join(sw_inc_dir, 'podofo')
diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py
index 7d8ea4d80a..e9e47816fd 100644
--- a/setup/installer/windows/freeze.py
+++ b/setup/installer/windows/freeze.py
@@ -18,7 +18,7 @@ QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
 LIBUSB_DIR       = 'C:\\libusb'
 LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
 SW               = r'C:\cygwin\home\kovid\sw'
-IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.5.6',
+IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.6.6',
         'VisualMagick', 'bin')
 
 VERSION = re.sub('[a-z]\d+', '', __version__)
diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst
index b9aef39657..5dfd956ce2 100644
--- a/setup/installer/windows/notes.rst
+++ b/setup/installer/windows/notes.rst
@@ -301,12 +301,14 @@ int projectType = MULTITHREADEDDLL;
 
 Run configure.bat in a  visual studio command prompt
 
+Run configure.exe generated by configure.bat
+
 Edit magick/magick-config.h
 
 Undefine ProvideDllMain and MAGICKCORE_X11_DELEGATE
 
 Now open VisualMagick/VisualDynamicMT.sln set to Release
-Remove the CORE_xlib project
+Remove the CORE_xlib and UTIL_Imdisplay project CORE_Magick++
 
 calibre
 ---------

From 302d0c35468633353e900b9d3ff3567c31a28a08 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 08:45:51 -0700
Subject: [PATCH 07/33] Sunday Times (UK) by DM. Fixes #7978 (The Sunday Times
 (UK))

---
 resources/recipes/sunday_times.recipe | 115 ++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)
 create mode 100644 resources/recipes/sunday_times.recipe

diff --git a/resources/recipes/sunday_times.recipe b/resources/recipes/sunday_times.recipe
new file mode 100644
index 0000000000..1f20f73cd9
--- /dev/null
+++ b/resources/recipes/sunday_times.recipe
@@ -0,0 +1,115 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.thesundaytimes.co.uk
+'''
+import urllib
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TimesOnline(BasicNewsRecipe):
+    title                 = 'The Sunday Times UK'
+    __author__            = 'Darko Miletic'
+    description           = 'news from United Kingdom and World'
+    language              = 'en_GB'
+    publisher             = 'Times Newspapers Ltd'
+    category              = 'news, politics, UK'
+    oldest_article        = 3
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    delay                 = 1
+    needs_subscription    = True
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://www.thesundaytimes.co.uk/sto/public/images/logos/logo-home.gif'
+    INDEX                 = 'http://www.thesundaytimes.co.uk'
+    PREFIX                = u'http://www.thesundaytimes.co.uk/sto/'
+    extra_css             = """
+                                .author-name,.authorName{font-style: italic}
+                                .published-date,.multi-position-photo-text{font-family: Arial,Helvetica,sans-serif;
+                                                                           font-size: small; color: gray;
+                                                                           display:block; margin-bottom: 0.5em}
+                                body{font-family: Georgia,"Times New Roman",Times,serif}
+                            """
+
+    conversion_options = {
+                          'comment'    : description
+                        , 'tags'       : category
+                        , 'publisher'  : publisher
+                        , 'language'   : language
+                        }
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open('http://www.timesplus.co.uk/tto/news/?login=false&url=http://www.thesundaytimes.co.uk/sto/')
+        if self.username is not None and self.password is not None:
+            data = urllib.urlencode({ 'userName':self.username
+                                     ,'password':self.password
+                                     ,'keepMeLoggedIn':'false'
+                                   })
+            br.open('https://www.timesplus.co.uk/iam/app/authenticate',data)
+        return br
+
+    remove_tags      = [
+                           dict(name=['object','link','iframe','base','meta'])
+                          ,dict(attrs={'class':'tools comments-parent' })
+                        ]
+    remove_attributes=['lang']
+    keep_only_tags   = [
+                          dict(attrs={'class':'standard-content'})
+                         ,dict(attrs={'class':'f-author'})
+                         ,dict(attrs={'id':'bodycopy'})
+                       ]
+    remove_tags_after=dict(attrs={'class':'tools_border'})
+
+    feeds = [
+                (u'UK News'     , PREFIX + u'news/uk_news/'        )
+               ,(u'World'       , PREFIX + u'news/world_news/'     )
+               ,(u'Politics'    , PREFIX + u'news/Politics/'       )
+               ,(u'Focus'       , PREFIX + u'news/focus/'          )
+               ,(u'Insight'     , PREFIX + u'news/insight/'        )
+               ,(u'Ireland'     , PREFIX + u'news/ireland/'        )
+               ,(u'Columns'     , PREFIX + u'comment/columns/'     )
+               ,(u'Arts'        , PREFIX + u'culture/arts/'        )
+               ,(u'Books'       , PREFIX + u'culture/books/'       )
+               ,(u'Film and TV' , PREFIX + u'culture/film_and_tv/' )
+               ,(u'Sport'       , PREFIX + u'sport/'               )
+               ,(u'Business'    , PREFIX + u'business'             )
+               ,(u'Money'       , PREFIX + u'business/money/'      )
+               ,(u'Style'       , PREFIX + u'style/'               )
+               ,(u'Travel'      , PREFIX + u'travel/'              )
+               ,(u'Clarkson'    , PREFIX + u'ingear/clarkson/'     )
+               ,(u'Cars'        , PREFIX + u'ingear/cars/'         )
+               ,(u'Bikes'       , PREFIX + u'ingear/2_Wheels/'     )
+               ,(u'Tech'        , PREFIX + u'ingear/Tech___Games/' )
+               ,(u'Magazine'    , PREFIX + u'Magazine/'            )
+            ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for atag in soup.findAll('a',href=True):
+                parentName = atag.parent.name
+                title      = self.tag_to_string(atag).strip()
+                if (parentName == 'h2' or parentName == 'h3') and title is not None and title != '':
+                    url           = self.INDEX + atag['href']
+                    articles.append({
+                                          'title'      :title
+                                         ,'date'       :''
+                                         ,'url'        :url
+                                         ,'description':''
+                                        })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds

From dab89f6f791a0ecb10b47dbb3b82ddf0d53235c8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 08:53:57 -0700
Subject: [PATCH 08/33] Fix bug when using tags like custom column in the
 template language

---
 src/calibre/library/save_to_disk.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py
index 3179551b45..20c0a7e59c 100644
--- a/src/calibre/library/save_to_disk.py
+++ b/src/calibre/library/save_to_disk.py
@@ -131,6 +131,8 @@ class SafeFormat(TemplateFormatter):
                 return self.composite_values[key]
             if key in kwargs:
                 val = kwargs[key]
+                if isinstance(val, list):
+                    val = ','.join(val)
                 return val.replace('/', '_').replace('\\', '_')
             return ''
         except:

From 9fcf243a72616a42c58c2e6071c42096f101d9c9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 08:55:52 -0700
Subject: [PATCH 09/33] Fix bug that prevent the Disabled option for Tag
 Browser partiotining from working in the Preferences dialog

---
 src/calibre/gui2/preferences/look_feel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py
index 263d19325d..37ed90cc61 100644
--- a/src/calibre/gui2/preferences/look_feel.py
+++ b/src/calibre/gui2/preferences/look_feel.py
@@ -57,7 +57,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
             (_('Never'), 'never')]
         r('toolbar_text', gprefs, choices=choices)
 
-        choices = [(_('Disabled'), 'disabled'), (_('By first letter'), 'first letter'),
+        choices = [(_('Disabled'), 'disable'), (_('By first letter'), 'first letter'),
                    (_('Partitioned'), 'partition')]
         r('tags_browser_partition_method', gprefs, choices=choices)
         r('tags_browser_collapse_at', gprefs)

From 7918f0c075a240f2fd06d781193017625afaf2e9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 08:57:20 -0700
Subject: [PATCH 10/33] Template language: Make all column names case
 insensitive

---
 src/calibre/ebooks/metadata/book/base.py | 5 +++--
 src/calibre/utils/formatter.py           | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index e3fb8092e6..77df6b00c2 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -38,15 +38,16 @@ class SafeFormat(TemplateFormatter):
 
     def get_value(self, key, args, kwargs):
         try:
+            key = key.lower()
             if key != 'title_sort':
-                key = field_metadata.search_term_to_field_key(key.lower())
+                key = field_metadata.search_term_to_field_key(key)
             b = self.book.get_user_metadata(key, False)
             if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
                 v = ''
             elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
                 v = ''
             else:
-                ign, v = self.book.format_field(key.lower(), series_with_index=False)
+                ign, v = self.book.format_field(key, series_with_index=False)
             if v is None:
                 return ''
             if v == '':
diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py
index 4e9710ad14..f4e687b419 100644
--- a/src/calibre/utils/formatter.py
+++ b/src/calibre/utils/formatter.py
@@ -468,6 +468,7 @@ class EvalFormatter(TemplateFormatter):
     A template formatter that uses a simple dict instead of an mi instance
     '''
     def get_value(self, key, args, kwargs):
+        key = key.lower()
         return kwargs.get(key, _('No such variable ') + key)
 
 eval_formatter = EvalFormatter()

From 8d38075e3b1b1491f8ff595ecd715dc8ec4c6431 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 09:01:05 -0700
Subject: [PATCH 11/33] Change search/replace to show commas instead of
 vertical bars as the separator for multiple authors

---
 src/calibre/gui2/dialogs/metadata_bulk.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 9dbc3dee5e..ef14c95b1d 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -436,7 +436,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             elif not fm['is_multiple']:
                 val = [val]
             elif field == 'authors':
-                val = [v.replace(',', '|') for v in val]
+                val = [v.replace('|', ',') for v in val]
         else:
             val = []
         return val
@@ -566,17 +566,11 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             dest_val = mi.get(dest, '')
             if dest_val is None:
                 dest_val = []
-            elif isinstance(dest_val, list):
-                if dest == 'authors':
-                    dest_val = [v.replace(',', '|') for v in dest_val]
-            else:
+            elif not isinstance(dest_val, list):
                 dest_val = [dest_val]
         else:
             dest_val = []
 
-        if len(val) > 0:
-            if src == 'authors':
-                val = [v.replace(',', '|') for v in val]
         if dest_mode == 1:
             val.extend(dest_val)
         elif dest_mode == 2:

From 08a59b31c2e19bd46a80b7ef2fc1130d1b47bc17 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 4 Jan 2011 21:04:49 +0000
Subject: [PATCH 12/33] Fix problem with EXTRA_CUSTOMIZATION lists when there
 was no customization before.

---
 src/calibre/devices/usbms/deviceconfig.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/calibre/devices/usbms/deviceconfig.py b/src/calibre/devices/usbms/deviceconfig.py
index 769543f7cc..940ea96f38 100644
--- a/src/calibre/devices/usbms/deviceconfig.py
+++ b/src/calibre/devices/usbms/deviceconfig.py
@@ -106,12 +106,13 @@ class DeviceConfig(object):
     def settings(cls):
         opts = cls._config().parse()
         if isinstance(cls.EXTRA_CUSTOMIZATION_DEFAULT, list):
+            if opts.extra_customization is None:
+                opts.extra_customization = []
             if not isinstance(opts.extra_customization, list):
                 opts.extra_customization = [opts.extra_customization]
-            else:
-                for i,d in enumerate(cls.EXTRA_CUSTOMIZATION_DEFAULT):
-                    if i >= len(opts.extra_customization):
-                        opts.extra_customization.append(d)
+            for i,d in enumerate(cls.EXTRA_CUSTOMIZATION_DEFAULT):
+                if i >= len(opts.extra_customization):
+                    opts.extra_customization.append(d)
         return opts
 
     @classmethod

From 91f78bf034a0716f4d95da2f017f1c80c5ef285c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Tue, 4 Jan 2011 14:04:52 -0700
Subject: [PATCH 13/33] Smarter Planet by Jack Mason

---
 resources/recipes/ibm_smarter_planet.recipe | 23 +++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 resources/recipes/ibm_smarter_planet.recipe

diff --git a/resources/recipes/ibm_smarter_planet.recipe b/resources/recipes/ibm_smarter_planet.recipe
new file mode 100644
index 0000000000..2e5c46fb80
--- /dev/null
+++ b/resources/recipes/ibm_smarter_planet.recipe
@@ -0,0 +1,23 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1293122276(BasicNewsRecipe):
+    title          = u'Smarter Planet | Tumblr for eReaders'
+    __author__ = 'Jack Mason'
+    author  = 'IBM Global Business Services'
+    publisher             = 'IBM'
+    category              = 'news, technology, IT, internet of things, analytics'
+    oldest_article = 7
+    max_articles_per_feed = 30
+    no_stylesheets = True
+    use_embedded_content = False
+    masthead_url          = 'http://30.media.tumblr.com/tumblr_l70dow9UmU1qzs4rbo1_r3_250.jpg'
+    remove_tags_before = dict(id='item')
+    remove_tags_after  = dict(id='item')
+    remove_tags = [dict(attrs={'class':['sidebar', 'about', 'footer', 'description,' 'disqus', 'nav', 'notes', 'disqus_thread']}),
+                dict(id=['sidebar', 'footer', 'disqus', 'nav', 'notes', 'likes_container', 'description', 'disqus_thread', 'about']),
+                dict(name=['script', 'noscript', 'style'])]
+
+
+
+    feeds          = [(u'Smarter Planet Tumblr', u'http://smarterplanet.tumblr.com/mobile/rss')]
+

From c793002de7e050750c87da49a24fd8adef753c3b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 08:39:39 +0000
Subject: [PATCH 14/33] Make evaluation of composite columns just-in-time,
 instead of just-in-case.

---
 src/calibre/library/caches.py | 74 +++++++++++++++++++++++++----------
 1 file changed, 53 insertions(+), 21 deletions(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index a32c45191f..3a61a8fd5d 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -132,6 +132,48 @@ def _match(query, value, matchkind):
             pass
     return False
 
+class CacheRow(object):
+
+    def __init__(self, db, composites, val):
+        self.db = db
+        self.composites = composites
+        self._mydata = val
+        self._must_do = len(composites) > 0
+
+    def __getitem__(self, col):
+        rec = self._mydata
+        if self._must_do and col in self.composites:
+            self._must_do = False
+            mi = self.db.get_metadata(rec[0], index_is_id=True)
+            for c in self.composites:
+                rec[c] = mi.get(self.composites[c])
+        return rec[col]
+
+    def __setitem__ (self, col, val):
+        self._mydata[col] = val
+
+    def append(self, val):
+        self._mydata.append(val)
+
+    def get(self, col, default):
+        try:
+            return self.__getitem__(col)
+        except:
+            return default
+
+    def __len__(self):
+        return len(self._mydata)
+
+    def __iter__(self):
+        for v in self._mydata:
+            yield v
+
+    def __str__(self):
+        return self.__unicode__()
+
+    def __unicode__(self):
+        return unicode(self._mydata)
+
 class ResultCache(SearchQueryParser): # {{{
 
     '''
@@ -139,7 +181,12 @@ class ResultCache(SearchQueryParser): # {{{
     '''
     def __init__(self, FIELD_MAP, field_metadata):
         self.FIELD_MAP = FIELD_MAP
-        self._map = self._data = self._map_filtered = []
+        self.composites = {}
+        for key in field_metadata:
+            if field_metadata[key]['datatype'] == 'composite':
+                self.composites[field_metadata[key]['rec_index']] = key
+        self._data = []
+        self._map = self._map_filtered = []
         self.first_sort = True
         self.search_restriction = ''
         self.field_metadata = field_metadata
@@ -148,10 +195,6 @@ class ResultCache(SearchQueryParser): # {{{
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
-        self.composites = []
-        for key in field_metadata:
-            if field_metadata[key]['datatype'] == 'composite':
-                self.composites.append((key, field_metadata[key]['rec_index']))
 
     def __getitem__(self, row):
         return self._data[self._map_filtered[row]]
@@ -583,13 +626,10 @@ class ResultCache(SearchQueryParser): # {{{
         '''
         for id in ids:
             try:
-                self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
+                self._data[id] = CacheRow(db, self.composites,
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
                 self._data[id].append(db.book_on_device_string(id))
                 self._data[id].append(None)
-                if len(self.composites) > 0:
-                    mi = db.get_metadata(id, index_is_id=True)
-                    for k,c in self.composites:
-                        self._data[id][c] = mi.get(k, None)
             except IndexError:
                 return None
         try:
@@ -603,13 +643,10 @@ class ResultCache(SearchQueryParser): # {{{
             return
         self._data.extend(repeat(None, max(ids)-len(self._data)+2))
         for id in ids:
-            self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
+            self._data[id] = CacheRow(db, self.composites,
+                        db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0])
             self._data[id].append(db.book_on_device_string(id))
             self._data[id].append(None)
-            if len(self.composites) > 0:
-                mi = db.get_metadata(id, index_is_id=True)
-                for k,c in self.composites:
-                    self._data[id][c] = mi.get(k)
         self._map[0:0] = ids
         self._map_filtered[0:0] = ids
 
@@ -630,16 +667,11 @@ class ResultCache(SearchQueryParser): # {{{
         temp = db.conn.get('SELECT * FROM meta2')
         self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else []
         for r in temp:
-            self._data[r[0]] = r
+            self._data[r[0]] = CacheRow(db, self.composites, r)
         for item in self._data:
             if item is not None:
                 item.append(db.book_on_device_string(item[0]))
                 item.append(None)
-                if len(self.composites) > 0:
-                    mi = db.get_metadata(item[0], index_is_id=True)
-                    for k,c in self.composites:
-                        item[c] = mi.get(k)
-
         self._map = [i[0] for i in self._data if i is not None]
         if field is not None:
             self.sort(field, ascending)

From 2662ab485fb978a27dcdc613b68e45613e8c6ac6 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 09:49:27 +0000
Subject: [PATCH 15/33] Make tags sorted in the meta2 table.

---
 src/calibre/gui2/library/models.py    |  4 ++--
 src/calibre/library/custom_columns.py |  9 +++++----
 src/calibre/library/database2.py      |  2 +-
 src/calibre/library/sqlite.py         | 23 ++++++++++++++++++++++-
 4 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 49cb1ce182..6a48aef9be 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -526,7 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def tags(r, idx=-1):
             tags = self.db.data[r][idx]
             if tags:
-                return QVariant(', '.join(sorted(tags.split(','), key=sort_key)))
+                return QVariant(', '.join(tags.split(',')))
             return None
 
         def series_type(r, idx=-1, siix=-1):
@@ -577,7 +577,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def text_type(r, mult=False, idx=-1):
             text = self.db.data[r][idx]
             if text and mult:
-                return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
+                return QVariant(', '.join(text.split('|')))
             return QVariant(text)
 
         def number_type(r, idx=-1):
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 07ea407460..558f3b8fc9 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -14,6 +14,7 @@ from calibre.constants import preferred_encoding
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import parse_date
 from calibre.utils.config import tweaks
+from calibre.utils.icu import sort_key
 
 class CustomColumns(object):
 
@@ -181,8 +182,8 @@ class CustomColumns(object):
         ans = row[self.FIELD_MAP[data['num']]]
         if data['is_multiple'] and data['datatype'] == 'text':
             ans = ans.split('|') if ans else []
-            if data['display'].get('sort_alpha', False):
-                ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
+            if data['display'].get('sort_alpha', True):
+                ans.sort(key=sort_key)
         return ans
 
     def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@@ -534,8 +535,8 @@ class CustomColumns(object):
             if data['normalized']:
                 query = '%s.value'
                 if data['is_multiple']:
-                    query = 'group_concat(%s.value, "|")'
-                    if not display.get('sort_alpha', False):
+                    query = 'cc_sortconcat(%s.value)'
+                    if not display.get('sort_alpha', True):
                         query = 'sort_concat(link.id, %s.value)'
                 line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 611aa1cc89..0b1c6a6cfb 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -242,7 +242,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'timestamp',
              '(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size',
             ('rating', 'ratings', 'rating', 'ratings.rating'),
-            ('tags', 'tags', 'tag', 'group_concat(name)'),
+            ('tags', 'tags', 'tag', 'tags_sortconcat(name)'),
              '(SELECT text FROM comments WHERE book=books.id) comments',
             ('series', 'series', 'series', 'name'),
             ('publisher', 'publishers', 'publisher', 'name'),
diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py
index 0458ada27b..0c3ae487ea 100644
--- a/src/calibre/library/sqlite.py
+++ b/src/calibre/library/sqlite.py
@@ -19,7 +19,7 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.date import parse_date, isoformat
 from calibre import isbytestring, force_unicode
 from calibre.constants import iswindows, DEBUG
-from calibre.utils.icu import strcmp
+from calibre.utils.icu import strcmp, sort_key
 
 global_lock = RLock()
 
@@ -69,6 +69,25 @@ class Concatenate(object):
             return None
         return self.sep.join(self.ans)
 
+class TagsSortConcatenate(object):
+    '''Sorted string concatenation aggregator for sqlite'''
+    def __init__(self, sep=','):
+        self.sep = sep
+        self.ans = []
+
+    def step(self, value):
+        if value is not None:
+            self.ans.append(value)
+
+    def finalize(self):
+        if not self.ans:
+            return None
+        return self.sep.join(sorted(self.ans, key=sort_key))
+
+class CcSortConcatenate(TagsSortConcatenate):
+    def __init__(self):
+        TagsSortConcatenate.__init__(self, sep='|')
+
 class SortedConcatenate(object):
     '''String concatenation aggregator for sqlite, sorted by supplied index'''
     sep = ','
@@ -155,6 +174,8 @@ class DBThread(Thread):
         c_ext_loaded = load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
+        self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate)
+        self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate)
         if not c_ext_loaded:
             self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
             self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)

From 84c6bcac39d515bda5e3344ad577d0ce214576d8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 12:59:18 +0000
Subject: [PATCH 16/33] Add ability to manipulate int, float, and bool columns
 in search replace

---
 src/calibre/gui2/dialogs/metadata_bulk.py |  5 ++++-
 src/calibre/library/custom_columns.py     | 22 +++++++++++++++++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index ef14c95b1d..e1ee4327f3 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -321,7 +321,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             if (f in ['author_sort'] or
                     (fm[f]['datatype'] in ['text', 'series', 'enumeration']
                      and fm[f].get('search_terms', None)
-                     and f not in ['formats', 'ondevice', 'sort'])):
+                     and f not in ['formats', 'ondevice', 'sort']) or
+                    fm[f]['datatype'] in ['int', 'float', 'bool'] ):
                 self.all_fields.append(f)
                 self.writable_fields.append(f)
             if f in ['sort'] or fm[f]['datatype'] == 'composite':
@@ -431,6 +432,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 val = mi.get('title_sort', None)
             else:
                 val = mi.get(field, None)
+            if isinstance(val, (int, float, bool)):
+                val = str(val)
             if val is None:
                 val = [] if fm['is_multiple'] else ['']
             elif not fm['is_multiple']:
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 558f3b8fc9..ccdd55021d 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -134,7 +134,15 @@ class CustomColumns(object):
 
         def adapt_bool(x, d):
             if isinstance(x, (str, unicode, bytes)):
-                x = bool(int(x))
+                x = x.lower()
+                if x == 'true':
+                    x = True
+                elif x == 'false':
+                    x = False
+                elif x == 'none':
+                    x = None
+                else:
+                    x = bool(int(x))
             return x
 
         def adapt_enum(x, d):
@@ -143,9 +151,17 @@ class CustomColumns(object):
                 v = None
             return v
 
+        def adapt_number(x, d):
+            if isinstance(x, (str, unicode, bytes)):
+                if x.lower() == 'none':
+                    return None
+            if d['datatype'] == 'int':
+                return int(x)
+            return float(x)
+
         self.custom_data_adapters = {
-                'float': lambda x,d : x if x is None else float(x),
-                'int':   lambda x,d : x if x is None else int(x),
+                'float': adapt_number,
+                'int':   adapt_number,
                 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),

From 3f3944e95b6a4fd7d830d98187b78a45298796e8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 14:51:14 +0000
Subject: [PATCH 17/33] Enhancement #8035: Advanced Search,
 Titel/Author?series/Tag - Type Ahead Word Lists

---
 src/calibre/gui2/dialogs/search.py | 44 +++++++++++++++++++++++++-----
 src/calibre/gui2/dialogs/search.ui | 27 ++++++++++++++----
 2 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/src/calibre/gui2/dialogs/search.py b/src/calibre/gui2/dialogs/search.py
index 8e8fd09652..62a0f8a9f1 100644
--- a/src/calibre/gui2/dialogs/search.py
+++ b/src/calibre/gui2/dialogs/search.py
@@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import re, copy
 
-from PyQt4.QtGui import QDialog, QDialogButtonBox
+from PyQt4.Qt import QDialog, QDialogButtonBox, QCompleter, Qt
 
 from calibre.gui2.dialogs.search_ui import Ui_Dialog
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
@@ -22,6 +22,28 @@ class SearchDialog(QDialog, Ui_Dialog):
                              key=lambda x: sort_key(x if x[0] != '#' else x[1:]))
         self.general_combo.addItems(searchables)
 
+        all_authors = db.all_authors()
+        all_authors.sort(key=lambda x : sort_key(x[1]))
+        for i in all_authors:
+            id, name = i
+            name = name.strip().replace('|', ',')
+            self.authors_box.addItem(name)
+        self.authors_box.setEditText('')
+        self.authors_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+        self.authors_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+        all_series = db.all_series()
+        all_series.sort(key=lambda x : sort_key(x[1]))
+        for i in all_series:
+            id, name = i
+            self.series_box.addItem(name)
+        self.series_box.setEditText('')
+        self.series_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+        self.series_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+        all_tags = db.all_tags()
+        self.tags_box.update_tags_cache(all_tags)
+
         self.box_last_values = copy.deepcopy(box_values)
         if self.box_last_values:
             for k,v in self.box_last_values.items():
@@ -121,26 +143,34 @@ class SearchDialog(QDialog, Ui_Dialog):
             return tok
 
     def box_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+
         ans = []
         self.box_last_values = {}
         title = unicode(self.title_box.text()).strip()
         self.box_last_values['title_box'] = title
         if title:
-            ans.append('title:"' + title + '"')
+            ans.append('title:"' + self.mc + title + '"')
         author = unicode(self.authors_box.text()).strip()
         self.box_last_values['authors_box'] = author
         if author:
-            ans.append('author:"' + author + '"')
+            ans.append('author:"' + self.mc + author + '"')
         series = unicode(self.series_box.text()).strip()
         self.box_last_values['series_box'] = series
         if series:
-            ans.append('series:"' + series + '"')
-        self.mc = '='
+            ans.append('series:"' + self.mc + series + '"')
+
         tags = unicode(self.tags_box.text())
         self.box_last_values['tags_box'] = tags
-        tags = self.tokens(tags)
+        tags = [t.strip() for t in tags.split(',') if t.strip()]
         if tags:
-            tags = ['tags:' + t for t in tags]
+            tags = ['tags:"=' + t + '"' for t in tags]
             ans.append('(' + ' or '.join(tags) + ')')
         general = unicode(self.general_box.text())
         self.box_last_values['general_box'] = general
diff --git a/src/calibre/gui2/dialogs/search.ui b/src/calibre/gui2/dialogs/search.ui
index 7bb4c15363..6848a45506 100644
--- a/src/calibre/gui2/dialogs/search.ui
+++ b/src/calibre/gui2/dialogs/search.ui
@@ -21,7 +21,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label_5">
      <property name="text">
-      <string>What kind of match to use:</string>
+      <string>&amp;What kind of match to use:</string>
      </property>
      <property name="buddy">
       <cstring>matchkind</cstring>
@@ -228,7 +228,7 @@
         </widget>
        </item>
        <item row="1" column="1">
-        <widget class="QLineEdit" name="title_box">
+        <widget class="EnLineEdit" name="title_box">
          <property name="toolTip">
           <string>Enter the title.</string>
          </property>
@@ -265,21 +265,21 @@
         </widget>
        </item>
        <item row="2" column="1">
-        <widget class="QLineEdit" name="authors_box">
+        <widget class="EnComboBox" name="authors_box">
          <property name="toolTip">
           <string>Enter an author's name. Only one author can be used.</string>
          </property>
         </widget>
        </item>
        <item row="3" column="1">
-        <widget class="QLineEdit" name="series_box">
+        <widget class="EnComboBox" name="series_box">
          <property name="toolTip">
           <string>Enter a series name, without an index. Only one series name can be used.</string>
          </property>
         </widget>
        </item>
        <item row="4" column="1">
-        <widget class="QLineEdit" name="tags_box">
+        <widget class="TagsLineEdit" name="tags_box">
          <property name="toolTip">
           <string>Enter tags separated by spaces</string>
          </property>
@@ -348,6 +348,23 @@
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>EnLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>EnComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>TagsLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+ </customwidgets>
  <tabstops>
   <tabstop>all</tabstop>
   <tabstop>phrase</tabstop>

From 281db1fb8186b33a152a795989b7a169997c9c27 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 08:47:32 -0700
Subject: [PATCH 18/33] Arabian Business by DM. Fixes #8199 (New recipe for
 Arabian Business newsportal in English)

---
 resources/images/news/arabian_business.png | Bin 0 -> 4213 bytes
 resources/recipes/arabian_business.recipe  |  86 +++++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 resources/images/news/arabian_business.png
 create mode 100644 resources/recipes/arabian_business.recipe

diff --git a/resources/images/news/arabian_business.png b/resources/images/news/arabian_business.png
new file mode 100644
index 0000000000000000000000000000000000000000..e9498309885eeda4078f3b6ad312232fbf5d4bb6
GIT binary patch
literal 4213
zcmY+HcQ_l|+s6~5W=pFzT1xv^MbxG=5_>&JRcVY`(b%I>qo~kj^s!Sr6tP+fVrwa-
zYNo9bv0^-^+G59e^ZUL3yzl2a=X|g0zOL^-=bZc8_lbj97@g<5$_W4f&YKwPTQOAO
zKa=+yV`WQ~=w!$_S2H91v;V^JaDS2!;RrB>2Qk$6ztgbCTl+u3-~b&YL;vds46eVm
z1ps)kCi*(oq2p@~;cp!t2%Y88v$OJW6Kn6LYJ7Y4fOViLjEz+RYmE72Jr#YBSI+BX
z!3tLe{guBi{S)P(B6uzCX7)k+_?X{M8@qyJsKMo89yu|eYB?{_7K7%bn*PB_t$l~H
zgQH!orNO*)`O13pXQ{zRQf~O*{&8edkp0uuv7P0hB}ooRjntRed#2@E{Z4|EuR`G_
zkqg80$8&$~o+#e48^jz2N*7=y$iywKdPs_kw6klly%PxeCME7y5WZi_4gBONKvyfk
zM}tP2#~n=Oit@!+PzycT&PiUJ$;uW8);P|$q)k7ZBT}N4uHQgZ0Z9Fy4(;Dn=dRE$
z*juIkD6MrL*$3Fc>7T**Hj`Oc2=xteQEuL)*TEN0P=}08BfU-|M9xCkO(9Xuw51kZ
zdFT)Lbpf&8a!K6mz4fZqf5d-|JG@VyPoDBV-{>cAD-D3!+naICH2eClt)b^psQz3I
zjloq45y^}tPfiZAc;tzj0!@BVqyA;*H6TIS{6!0P+h4HpFxiP}Pf~dRE`qOhFM?S$
zF0Apf$lBr^I2zYiDH)GArEzsu=urf3qTY_8tx4f!8shY4_L3)V^aTU-C>Ts=<?P&o
zoc^5P>fCBOJ0U~?O}6(Y7L>r(x;v8Q0Vr(o0G_K+en<zEXN_)37Ib?o;k_XIj>Rv-
zP^Zd=Qb%TLjXK%w%V5SfAbK#iOxT8i*%(#;OfJ>+w{xYZB~@TViX)kn(dWW${ekdl
zyH^$}=tn*M{udOtP0RNmgT|LxBR*RlM(8om(A6iaBJRx4$&Mjwbvs~1nE{<eg|(*s
zgz|@2b)?5s`wOE)z`}N-h*mJ*UajZQI}lWOgW49=R_T#?<KCC;;cVn`lGGmdf!gr7
zcJX^Mi9oWiZ=TwD_Lz|mws@8&P4@mLzm0&naRvaK1DWQNq;%ymXXhhKA1iQ%zVST7
zR$!(%!%;0nKI`QY5YwCVd?zhGb~F7Z&*kP<bEv_A7zEE_J*9kW)60lwu@P9sqR%4I
zm;G9N2d;kEZ)w)licU#+y3%^|0J<{Gz|MfE!`>!!S0j-KlwK3E3&+j_bgEpt;;%s6
z<_S?J#C+hZj;WBS^&s?oZ9_vtPK^5J&bz6D&Sv-GBjxB0xU1I^3<_Y4Cwr>iXtB_Z
zaQ{y!voXW)O3va;?Hqhw7_wUDj!Bub8IrX}eXkY2LPsw-J{_6sg6e&B44Dcm%~FXi
ztW$e$er!^1&5K9UP`UFO7dVdbEdX*PxkbLEfB0i#)-84%yVcymh%JvWAQcfB8v4*8
zaB6fk;r2O)kilFy(c{}wUsGdfB?I)zrC|U`n@(=aHdLNoPYLk(HIQ!(+@?`d1BHhQ
zVrEV@oT{0mG4WmB452G2DGk#~%MBs(Nbq5Yp)nslZcO2$O|<EPZQId+KU#NI)(PHl
z0anozwv9+zZlKCz&<YW&)!m|O9h>cJ4SuY(+Nj@~E`Ty?nJdcJ%LxLGEdSEH$<xoZ
zScj4#I{D>3^acXDvs6D6F$YYUq)D19r7=%@{RQiMmn(j9a(2iAlcz?VG4lWdCx?hp
zZJpvIZ|@w<E9@~m5i=z@0BUJI91~ZW>{|UE7~mKqePNvG#a!I#MfAm&-n+h{Uz6U<
zj#l!3SpfG-RMeRdkEI}C6TZ;8qf*Gp!fj~N@fT*u$t}gvA-9TQZd7O0%CDtX?E>qg
zMfEnCh&}M{1P&p*R(P-xTOP!nGg(}Z!QvkjU0%fPJD_4D<I%H2cd_{3wein6gD_5Z
zy=OPM8+<6OBAF`7aZ=Zx3kFOz1>NGuNx0^JNZ(j`vKY5rx%RV|o%G21TH`PIqNhy`
z^30gt67w8&=D>)}nfH27VaVaX_bpS{-o72S6C-5`0Z#@M9|vq61Of_bc|82EO0Tes
zGX}bs>V?$$Q?zAI9J58cFX}npR(Rnj3^_aIx(!Iw>p_1vQusQlm}n*ugB>4Q6N|bD
z9=th?C0?}2)Cl%*2E-QnPFxRT`H{`)ubeF|2i3~N5&4$%&98SyMyBs4%t$EKf}VXW
zd`lF!zg0{4ff*#Te*azkje*K;ozl=V50B8yKLRDx^cVBPaX`;PZ?luyulICZp6k8S
z<F~V)NjB3Qto3BvG%FBjU;w@HDC(rf9li9F0kAb{0h7QFG&RrDeCkyX&rC<aL*Gr|
zfIWf2QtSY-ucxOsbFE~5u%i|@c*-DwAH)<h^X(d;WpFy@MoQIP?jm;fuTG~=zMc~^
z=?FkskH*EUdi2$Gyu1;+gb2bT4k-@u`D_tUr^kJ_+3s#cZrE2dZGOU;5t`$-kM0Bk
zaM{`YGa!Ch%*}jBgV`~4S2vwxtNW#Ds=%g6BqY3B1j@_J!-v>ROMlh0zK}u6Vg^|M
z&N>H$$YP|K3V&~P=Z7+tSvR+cM!^20sx0PUy$8eLofAQ7EY%vu^qLz;saGAPkjQQk
z^oCR9ZmI;@N*Otrsk7%i%2ucIR&X9{p5e|;9Ga|V(@QSr#cdzz0yjSm4jLqkX%f>j
zr`Rk~qodn(3ak#hZmanKWH8jz;CVx_L^?g`x@vwID3Ie$v6aQ8FY_p47L%K!<KLWd
z#^T(nrsi2nVM@t;m2g^|CE!&DQkxhI0$k=Ceo_@~pc^|ntb8Y;kzn}@l#5|$Kd&xl
ze@?t-XIUAUy;@9=^!2_<(Czp09XE@$h&x|jr?j)XzrW8XFd~DSWt`spr^Cf+Hx`}8
zE*^`<C9`>Qof-!?vnPBTjdLi+OA+_^VA7LeOQqVZEbSE!M9(w;Dk@bozFaaHMn=(I
zyIlfjry*kT*s#T>CA7x#Kjn}!q1ZxZwnyRTVev7|8~)T*cP>COHM2I8o@lgLT2TeR
zxZGS?Z;Md(6h^kkuyUbazYe1<4lq}Y<oLHPOFE-aCr9*k=0cC|#O~`wk>6pyJOW+V
zGUnnHWX4QCqfEAcWUoJP@EfVH_h&_aGwd+_c^j;*X;%&?Dxw~Ze%@ztX$)FQvt$(A
zJ5P3`3#<i8%uxL7LoUG*zOwx7xuzE>kyGvM(ZB*LN{#!3nhY=f$m2@86E7^ebuk-f
zR^DSY05)XWW2txv_CGxiqoG~5`kWp$T+n!aoD>yt<Y_`S&&qDkINJ;T>k%wR4X1i`
zNTNnIELXVRFk#ZaXGw}?hL)p9U?=)jSf%#<Bo<8JhTTshmZEgh<lp~-DxKy4gK(qW
zNo~9Qu=2Y*KZ=r=0zY|^@s4n8z}X4pR1qdO9YWo2Wn*avHB3ejYo~;o;eLsFm6xe!
zn{OZojO#<Gav3TaUp3-BeVKdbcNUihmm+O9;?zyFh*Oo$BxPb~XeuEV-`o<luLWKC
z9PaeYgpo*;UmBx`JFT7#dnCIzct}k!uU!qj&2#m_SIDu;gj$*||0e`;F|4%>(KFZe
zq+)(74_&(|Ovd!TNKy=!5k@#P5NYwUe<xUqTIfi}?=c!5wja%Q=k@qxyk6vl;n|Ik
zUctG?UABffzi;QEBG)w#eIEt0m1kWM2!y_v64#J<Y*6mro|(Mj`_%nD`JG(x*$$|+
zvu%NZqqUsrkxgIbWy2j|sefW-ck*g4t!isMg@Xx2MMW>RQ58Ha@fuo+Ze0fv1@ER&
zdwkt>SJZU4f3AetI(<fY+Bl}{!PSzy9*B!k5O%(A0a(h$qA$f@;CWnF?$qLfY+FxS
z&;7-L94&z|8>i6OE&++(#`Rt5`Vnb$5r?W|*hy6|=t2KbDOY((8)fIM2y4xif5WUj
z?veQ#cYX;ezw~cy>RZ4!783csFW3-D<SaQYPC-*v?`?Wl2yLg$ERXP2Yu*l9v1uVT
z!7uv7u{mz(BV*16u7zfKC6ZpU1UT`TYO*-as!0x0$Y`VF*$z%SUeT7ATci3wqDQ>E
zEW@dQ@o_tQO9_qY+-G4C;J$_%b8`vy_R+et2~Niy0%&m|5fKqp;uR_n)0e>Ck9jgx
z${*J58|$S|91`ZIT5c^T%cU->P0YPjxjyl*SuCw~^I&V%g^`(&ds#>T$^KL0hd%Mw
zvUeXB5QAoQ3#zx?Kpfh%KMa7>+`3;zN58)zW)kydABc<Za#b|T1u;NdIfQVZzbEk2
z$KVG$6RD^-j}!dJt-<=>&q%8q?^k6vSSn!T&-zsOzSTAQPa|NwyqZX`BeYE`f^dI=
zm{F3l!sIeGbwZ~yn{0o(=KrZ(oNV`zf;V@2YSTm5{aNJS%8}l;LOz>eo9o!U8%>Z-
zdnvbTUG(U>yG<bzC|IO0T}8y)Pv0i+qbhIr?Y7Aw=5$l)>0U^ix2$~SUF8+ctZAC}
zHTY;p4h&Y<w`jFI9o}{p>2-sf8^JNXPRl{$C<p$H2`t~)B}C0%gW_re-9pp>$x_Dr
z`rJ67trIx$Hz)D*J6<f5C;CEGx#;-NF{W>7icza^zcN}G2@I`k4%widKHl~YPltEs
z^u$T^xKwVgk!oE8{U#g3lwp<Y6Ak{N$s~eZo!6C_u#GStowSf`bFP{K=?a@@jrFyY
zB}laNcvH}lMRP@Qj_ULL*A6vSl;+*pHyrUXU;4?n>@V>*F_n_7$1W9pr~SIkTKq~q
z<zD4FMK5M;EpknjaeU98k>#N)t*v@*R+%j9Hn;Bz)Ot!_bLXRqGcG9Yva7dcgg64o
zpZ`tu94brSk9-2}wJC$(8QVJvLUbHR#>-8%en}=hUi8TyAYjJUygj`qe%zm!zD`QE
zK#wkJTg{HT()C{lO8IygP;`ts-3W?Qnb`SL#!DUx5Y+vfOEJW~%=O%X>J`4CJC-5B
zEl)<WYi?IJqdxrQm_I!F86G0C9A<J;+Rh7f+wLFS_pMdP;>1rlu6c4hmNm$-o4a<>
zpbpc=lz#N(o9UsaP4Dt_f>3^_lkzLPRN2SiFXbBe&#9QbW_SSZy~ii|HG*VsM(Jy@
zo~x^J?ec2H%pOh^$ldX^#cN#ez|pi2VDYMb^D4gF#Ijz;ovV+##M1!rpmW~;{J2{O
zuQ)hWrDpt{r(MyM=an)AR<C-IMtFwWxg49Og!)-n%|CXy-b2ctvOIw#lHYU|Hdv;t
zT>b2Vn6y?l65O@>vdYyyH7{u*#%!YLSkhG`i$Al+&vnB48>D|G8&iVyB5gro@ji#S
ziOU;u-%7oi4<kwzN&A6TKsUV~T3TOLGmD3Mloj)z+JLVPb{!5K5I7|@Psu+2kQPXF
zhg`g<cipV)ntil0)`IhJhicf3e8#t30%h<Jg>XT+Yq~vgX9z$^K~WK;z!<953L2U!
zs+y__AO&?z1qD*bn9Tn(`1&I}ABX*4gQ9|_;vG$8wf{9VY~MU)7yu>)7W#Ny=a~Ni
D1%5d$

literal 0
HcmV?d00001

diff --git a/resources/recipes/arabian_business.recipe b/resources/recipes/arabian_business.recipe
new file mode 100644
index 0000000000..8b41c99e68
--- /dev/null
+++ b/resources/recipes/arabian_business.recipe
@@ -0,0 +1,86 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.arabianbusiness.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Arabian_Business(BasicNewsRecipe):
+    title                 = 'Arabian Business'
+    __author__            = 'Darko Miletic'
+    description           = 'Comprehensive Guide to Middle East Business & Gulf Industry News including,Banking & Finance,Construction,Energy,Media & Marketing,Real Estate,Transportation,Travel,Technology,Politics,Healthcare,Lifestyle,Jobs & UAE guide.Top Gulf & Dubai Business News.'
+    publisher             = 'Arabian Business Publishing Ltd.'
+    category              = 'ArabianBusiness.com,Arab Business News,Middle East Business News,Middle East Business,Arab Media News,Industry Events,Middle East Industry News,Arab Business Industry,Dubai Business News,Financial News,UAE Business News,Middle East Press Releases,Gulf News,Arab News,GCC Business News,Banking Finance,Media Marketing,Construction,Oil Gas,Retail,Transportation,Travel Hospitality,Photos,Videos,Life Style,Fashion,United Arab Emirates,UAE,Dubai,Sharjah,Abu Dhabi,Qatar,KSA,Saudi Arabia,Bahrain,Kuwait,Oman,Europe,South Asia,America,Asia,news'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://www.arabianbusiness.com/skins/ab.main/gfx/arabianbusiness_logo_sm.gif'
+    extra_css             = """
+                               body{font-family: Georgia,serif }
+                               img{margin-bottom: 0.4em; margin-top: 0.4em; display:block}
+                               .byline,.dateline{font-size: small; display: inline; font-weight: bold}
+                               ul{list-style: none outside none;}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags_before=dict(attrs={'id':'article-title'})
+    remove_tags = [
+                     dict(name=['meta','link','base','iframe','embed','object'])
+                    ,dict(attrs={'class':'printfooter'})
+                  ]
+    remove_attributes=['lang']
+
+
+    feeds = [
+              (u'Africa'           , u'http://www.arabianbusiness.com/world/Africa/?service=rss'                 )
+             ,(u'Americas'         , u'http://www.arabianbusiness.com/world/americas/?service=rss'               )
+             ,(u'Asia Pacific'     , u'http://www.arabianbusiness.com/world/asia-pacific/?service=rss'           )
+             ,(u'Europe'           , u'http://www.arabianbusiness.com/world/europe/?service=rss'                 )
+             ,(u'Middle East'      , u'http://www.arabianbusiness.com/world/middle-east/?service=rss'            )
+             ,(u'South Asia'       , u'http://www.arabianbusiness.com/world/south-asia/?service=rss'             )
+             ,(u'Banking & Finance', u'http://www.arabianbusiness.com/industries/banking-finance/?service=rss'   )
+             ,(u'Construction'     , u'http://www.arabianbusiness.com/industries/construction/?service=rss'      )
+             ,(u'Education'        , u'http://www.arabianbusiness.com/industries/education/?service=rss'         )
+             ,(u'Energy'           , u'http://www.arabianbusiness.com/industries/energy/?service=rss'            )
+             ,(u'Healthcare'       , u'http://www.arabianbusiness.com/industries/healthcare/?service=rss'        )
+             ,(u'Media'            , u'http://www.arabianbusiness.com/industries/media/?service=rss'             )
+             ,(u'Real Estate'      , u'http://www.arabianbusiness.com/industries/real-estate/?service=rss'       )
+             ,(u'Retail'           , u'http://www.arabianbusiness.com/industries/retail/?service=rss'            )
+             ,(u'Technology'       , u'http://www.arabianbusiness.com/industries/technology/?service=rss'        )
+             ,(u'Transport'        , u'http://www.arabianbusiness.com/industries/transport/?service=rss'         )
+             ,(u'Travel'           , u'http://www.arabianbusiness.com/industries/travel-hospitality/?service=rss')
+             ,(u'Equities'         , u'http://www.arabianbusiness.com/markets/equities/?service=rss'             )
+             ,(u'Commodities'      , u'http://www.arabianbusiness.com/markets/commodities/?service=rss'          )
+             ,(u'Currencies'       , u'http://www.arabianbusiness.com/markets/currencies/?service=rss'           )
+             ,(u'Market Data'      , u'http://www.arabianbusiness.com/markets/market-data/?service=rss'          )
+             ,(u'Comment'          , u'http://www.arabianbusiness.com/opinion/comment/?service=rss'              )
+             ,(u'Think Tank'       , u'http://www.arabianbusiness.com/opinion/think-tank/?service=rss'           )
+             ,(u'Arts'             , u'http://www.arabianbusiness.com/lifestyle/arts/?service=rss'               )
+             ,(u'Cars'             , u'http://www.arabianbusiness.com/lifestyle/cars/?service=rss'               )
+             ,(u'Food'             , u'http://www.arabianbusiness.com/lifestyle/food/?service=rss'               )
+             ,(u'Sport'            , u'http://www.arabianbusiness.com/lifestyle/sport/?service=rss'              )
+            ]
+
+    def print_version(self, url):
+        return url + '?service=printer&page='
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup

From af5d7eef227952e70eb0284f312fe121ff979595 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 08:49:03 -0700
Subject: [PATCH 19/33] Deia by Gerardo Diez. Fixes #405 (New news feed)

---
 resources/recipes/deia.recipe | 70 +++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 resources/recipes/deia.recipe

diff --git a/resources/recipes/deia.recipe b/resources/recipes/deia.recipe
new file mode 100644
index 0000000000..980d59d3d1
--- /dev/null
+++ b/resources/recipes/deia.recipe
@@ -0,0 +1,70 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__author__    = 'Gerardo Diez'
+__copyright__ = 'Gerardo Diez<gerardo.diez.garcia@gmail.com>'
+description   = 'Main daily newspaper from Spain - v1.00 (05, Enero 2011)'
+__docformat__ = 'restructuredtext en'
+
+'''
+deia.com
+'''
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class Deia(BasicNewsRecipe):
+	title			='Deia'
+	__author__		='Gerardo Diez'
+	publisher		='Editorial Iparraguirre, S.A'
+	category             	='news, politics, finances, world, spain, euskadi'
+	publication_type      	='newspaper'
+	oldest_article		=1
+	max_articles_per_feed	=100
+	simultaneous_downloads	=10
+	cover_url		='http://2.bp.blogspot.com/_RjrWzC6tI14/TM6jrPLaBZI/AAAAAAAAFaI/ayffwxidFEY/s1600/2009-10-13-logo-deia.jpg'
+	timefmt			='[%a, %d %b, %Y]'
+	encoding		='utf8'
+	language		='es_ES'
+	remove_javascript	=True
+	remove_tags_after	=dict(id='Texto')
+	remove_tags_before	=dict(id='Texto')
+	remove_tags		=[dict(name='div', attrs={'class':['Herramientas ', 'Multimedia']})]
+	no_stylesheets		=True
+	extra_css		='h1 {margin-bottom: .15em;font-size: 2.7em; font-family: Georgia, "Times New Roman", Times, serif;} .Antetitulo {margin: 1em 0;text-transform: uppercase;color: #999;} .PieFoto {margin: .1em 0;padding: .5em .5em .5em .5em;background: #F0F0F0;} .PieFoto p {margin-bottom: 0;font-family: Georgia,"Times New Roman",Times,serif;font-weight: bold; font-style: italic; color: #666;}'
+	keep_only_tags		=[dict(name='div', attrs={'class':['Texto ', 'NoticiaFicha ']})]
+	feeds          		= [
+					(u'Bizkaia' ,u'http://www.deia.com/index.php/services/rss?seccion=bizkaia'),
+					(u'Bilbao' ,u'http://www.deia.com/index.php/services/rss?seccion=bilbao'),
+					(u'Hemendik eta Handik' ,u'http://www.deia.com/index.php/services/rss?seccion=hemendik-eta-handik'),
+					(u'Margen Derecha' ,u'http://www.deia.com/index.php/services/rss?seccion=margen-derecha'),
+					(u'Encartaciones y Margen Izquierda' ,u'http://www.deia.com/index.php/services/rss?seccion=margen-izquierda-encartaciones'),
+					(u'Costa' ,u'http://www.deia.com/index.php/services/rss?seccion=costa'),
+					(u'Duranguesado' ,u'http://www.deia.com/index.php/services/rss?seccion=duranguesado'),
+					(u'Llodio-Nervión' ,u'http://www.deia.com/index.php/services/rss?seccion=llodio-nervion'),
+					(u'Arratia-Nervión' ,u'http://www.deia.com/index.php/services/rss?seccion=arratia-nervion'),
+					(u'Uribe-Txorierri' ,u'http://www.deia.com/index.php/services/rss?seccion=uribe-txorierri'),
+					(u'Ecos de sociedad' ,u'http://www.deia.com/index.php/services/rss?seccion=ecos-de-sociedad'),
+					(u'Sucesos' ,u'http://www.deia.com/index.php/services/rss?seccion=sucesos'),
+					(u'Política' ,u'http://www.deia.com/index.php/services/rss?seccion=politica'),
+					(u'Euskadi' ,u'http://www.deia.com/index.php/services/rss?seccion=politica/euskadi'),
+					(u'España' ,u'http://www.deia.com/index.php/services/rss?seccion=politica/espana'),
+					(u'Sociedad',u'http://www.deia.com/index.php/services/rss?seccion=sociedad'),
+					(u'Euskadi' ,u'http://www.deia.com/index.php/services/rss?seccion=socidad/euskadi'),
+					(u'Sociedad.España' ,u'http://www.deia.com/index.php/services/rss?seccion=sociedad/espana'),
+					(u'Ocio y Cultura' ,u'http://www.deia.com/index.php/services/rss?seccion=ocio-y-cultura'),
+					#(u'Cultura' ,u'http://www.deia.com/index.php/services/rss?seccion=cultura'),
+					#(u'Ocio' ,u'http://www.deia.com/index.php/services/rss?seccion=ocio'),
+					(u'On' ,u'http://www.deia.com/index.php/services/rss?seccion=on'),
+					(u'Agenda' ,u'http://www.deia.com/index.php/services/rss?seccion=agenda'),
+					(u'Comunicación' ,u'http://www.deia.com/index.php/services/rss?seccion=comunicacion'),
+					(u'Viajes' ,u'http://www.deia.com/index.php/services/rss?seccion=viajes'),
+					(u'¡Mundo!' ,u'http://www.deia.com/index.php/services/rss?seccion=que-mundo'),
+					(u'Humor' ,u'http://www.deia.com/index.php/services/rss?seccion=humor'),
+					(u'Opinión' ,u'http://www.deia.com/index.php/services/rss?seccion=opinion'),
+					(u'Editorial' ,u'http://www.deia.com/index.php/services/rss?seccion=editorial'),
+					(u'Tribuna abierta' ,u'http://www.deia.com/index.php/services/rss?seccion=tribuna-abierta'),
+					(u'Colaboración' ,u'http://www.deia.com/index.php/services/rss?seccion=colaboracion'),
+					(u'Columnistas' ,u'http://www.deia.com/index.php/services/rss?seccion=columnistas'),
+					(u'Deportes' ,u'http://www.deia.com/index.php/services/rss?seccion=deportes'),
+					(u'Athletic' ,u'http://www.deia.com/index.php/services/rss?seccion=athletic'),
+					(u'Economía' ,'http://www.deia.com/index.php/services/rss?seccion=economia'),
+					(u'Mundo' ,u'http://www.deia.com/index.php/services/rss?seccion=mundo')]
+

From 164ef37cc8955d7bb8562201f416eee43470bcae Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 09:41:44 -0700
Subject: [PATCH 20/33] Beginning of a libwmf wrapper

---
 src/calibre/utils/wmf/__init__.py |   9 ++
 src/calibre/utils/wmf/wmf.c       | 200 ++++++++++++++++++++++++++++++
 2 files changed, 209 insertions(+)
 create mode 100644 src/calibre/utils/wmf/__init__.py
 create mode 100644 src/calibre/utils/wmf/wmf.c

diff --git a/src/calibre/utils/wmf/__init__.py b/src/calibre/utils/wmf/__init__.py
new file mode 100644
index 0000000000..68dfb8d2b5
--- /dev/null
+++ b/src/calibre/utils/wmf/__init__.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+
diff --git a/src/calibre/utils/wmf/wmf.c b/src/calibre/utils/wmf/wmf.c
new file mode 100644
index 0000000000..1f8e8a27f3
--- /dev/null
+++ b/src/calibre/utils/wmf/wmf.c
@@ -0,0 +1,200 @@
+#define UNICODE
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include <libwmf/api.h>
+#include <libwmf/svg.h>
+
+typedef struct {
+    char *data;
+    size_t len;
+    size_t pos;
+} buf;
+
+//This code is taken mostly from the Abiword wmf plugin
+
+
+// returns unsigned char cast to int, or EOF
+static int wmf_WMF_read(void * context) {
+    char c;
+	buf *info = (buf*)context;
+
+	if (info->pos == info->len)
+		return EOF;
+
+	c = info->data[pos];
+
+	info->pos++;
+
+	return (int)c;
+}
+
+// returns (-1) on error, else 0
+static int wmf_WMF_seek(void * context, long pos) {
+	buf* info = (buf*) context;
+
+    if (pos < 0 || (size_t)pos > info->len) return -1;
+	info->pos = (size_t)pos;
+	return 0;
+}
+
+// returns (-1) on error, else pos
+static long wmf_WMF_tell(void * context) {
+	buf* info = (buf*) context;
+
+	return (long) info->pos;
+}
+
+
+#define CLEANUP if(API) { if (stream) wmf_free(API, stream); wmf_api_destroy(API); };
+
+static PyObject *
+wmf_render(PyObject *self, PyObject *args) {
+    char *data;
+    Py_ssize_t sz;
+    PyObject *ans;
+
+	unsigned int disp_width  = 0;
+	unsigned int disp_height = 0;
+
+	float wmf_width;
+	float wmf_height;
+	float ratio_wmf;
+	float ratio_bounds;
+
+	unsigned long flags;
+
+	unsigned int max_width  = 1600;
+	unsigned int max_height = 1200;
+	unsigned long max_flags = 0;
+
+	static const char* Default_Description = "wmf2svg";
+
+	wmf_error_t err;
+
+	wmf_svg_t* ddata = 0;
+
+	wmfAPI* API = 0;
+	wmfD_Rect bbox;
+
+	wmfAPI_Options api_options;
+
+    buf read_info;
+
+	char *stream = NULL;
+	unsigned long stream_len = 0;
+
+    if (!PyArg_ParseTuple(args, "s#", &data, &sz))
+        return NULL;
+
+	flags = WMF_OPT_IGNORE_NONFATAL | WMF_OPT_FUNCTION;
+	api_options.function = wmf_svg_function;
+
+	err = wmf_api_create(&API, flags, &api_options);
+
+	if (err != wmf_E_None) {
+        CLEANUP;
+        return PyErr_NoMemory();
+	}
+
+	read_info.data = data;
+	read_info.len = sz;
+	read_info.pos = 0;
+
+	err = wmf_bbuf_input(API, wmf_WMF_read, wmf_WMF_seek, wmf_WMF_tell, (void *) &read_info);
+	if (err != wmf_E_None) {
+        CLEANUP;
+        PyErr_SetString(PyExc_Exception, "Failed to initialize WMF input");
+        return NULL;
+	}
+
+	err = wmf_scan(API, 0, &(bbox));
+	if (err != wmf_E_None)
+	{	
+        CLEANUP;
+        PyErr_SetString(PyExc_ValueError, "Failed to scan the WMF");
+        return NULL;
+	}
+
+/* Okay, got this far, everything seems cool.
+ */
+	ddata = WMF_SVG_GetData (API);
+
+	ddata->out = wmf_stream_create(API, NULL);
+
+	ddata->Description = (char *)Default_Description;
+
+	ddata->bbox = bbox;
+
+	wmf_display_size(API, &disp_width, &disp_height, 96, 96);
+
+	wmf_width  = (float) disp_width;
+	wmf_height = (float) disp_height;
+
+	if ((wmf_width <= 0) || (wmf_height <= 0)) {
+        CLEANUP;
+        PyErr_SetString(PyExc_ValueError, "Bad WMF image size");
+        return NULL;
+	}
+
+	if ((wmf_width  > (float) max_width )
+	 || (wmf_height > (float) max_height)) {
+		ratio_wmf = wmf_height / wmf_width;
+		ratio_bounds = (float) max_height / (float) max_width;
+
+		if (ratio_wmf > ratio_bounds) {
+			ddata->height = max_height;
+			ddata->width  = (unsigned int) ((float) ddata->height / ratio_wmf);
+		}
+		else {
+			ddata->width  = max_width;
+			ddata->height = (unsigned int) ((float) ddata->width  * ratio_wmf);
+		}
+	}
+	else {
+		ddata->width  = (unsigned int) ceil ((double) wmf_width );
+		ddata->height = (unsigned int) ceil ((double) wmf_height);
+	}
+
+	ddata->flags |= WMF_SVG_INLINE_IMAGES;
+
+	ddata->flags |= WMF_GD_OUTPUT_MEMORY | WMF_GD_OWN_BUFFER;
+
+    err = wmf_play(API, 0, &(bbox));
+
+    if (err != wmf_E_None) {
+        CLEANUP;
+        PyErr_SetString(PyExc_ValueError, "Playing of the WMF file failed");
+        return NULL;
+    }
+
+	wmf_stream_destroy(API, ddata->out, &stream, &stream_len);
+
+    ans = Py_BuildValue("s#", stream, stream_len);
+    
+    wmf_free(API, stream);
+    wmf_api_destroy (API);
+
+    return ans;
+}
+
+
+static PyMethodDef wmf_methods[] = {
+    {"render", wmf_render, METH_VARARGS,
+        "render(path) -> Render wmf as svg."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+
+PyMODINIT_FUNC
+initwmf(void) 
+{
+    PyObject* m;
+    m = Py_InitModule3("wmf", wmf_methods,
+                       "Wrapper for the libwmf library");
+
+
+}
+

From 7e309d5d28c159c2b2f425af392937f25ff256a1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 11:00:24 -0700
Subject: [PATCH 21/33] Search replace: Add ability to manipulate number and
 boolean columns. Add type ahead completion to the advanced search dialog.
 Fixes #8035 (Advanced Search, Titel/Author?series/Tag - Type Ahead Word
 Lists)

---
 src/calibre/gui2/dialogs/metadata_bulk.py |  5 ++-
 src/calibre/gui2/dialogs/search.py        | 44 +++++++++++++++++++----
 src/calibre/gui2/dialogs/search.ui        | 27 +++++++++++---
 src/calibre/library/custom_columns.py     | 22 ++++++++++--
 4 files changed, 82 insertions(+), 16 deletions(-)

diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index ef14c95b1d..e1ee4327f3 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -321,7 +321,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             if (f in ['author_sort'] or
                     (fm[f]['datatype'] in ['text', 'series', 'enumeration']
                      and fm[f].get('search_terms', None)
-                     and f not in ['formats', 'ondevice', 'sort'])):
+                     and f not in ['formats', 'ondevice', 'sort']) or
+                    fm[f]['datatype'] in ['int', 'float', 'bool'] ):
                 self.all_fields.append(f)
                 self.writable_fields.append(f)
             if f in ['sort'] or fm[f]['datatype'] == 'composite':
@@ -431,6 +432,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
                 val = mi.get('title_sort', None)
             else:
                 val = mi.get(field, None)
+            if isinstance(val, (int, float, bool)):
+                val = str(val)
             if val is None:
                 val = [] if fm['is_multiple'] else ['']
             elif not fm['is_multiple']:
diff --git a/src/calibre/gui2/dialogs/search.py b/src/calibre/gui2/dialogs/search.py
index 8e8fd09652..62a0f8a9f1 100644
--- a/src/calibre/gui2/dialogs/search.py
+++ b/src/calibre/gui2/dialogs/search.py
@@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import re, copy
 
-from PyQt4.QtGui import QDialog, QDialogButtonBox
+from PyQt4.Qt import QDialog, QDialogButtonBox, QCompleter, Qt
 
 from calibre.gui2.dialogs.search_ui import Ui_Dialog
 from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
@@ -22,6 +22,28 @@ class SearchDialog(QDialog, Ui_Dialog):
                              key=lambda x: sort_key(x if x[0] != '#' else x[1:]))
         self.general_combo.addItems(searchables)
 
+        all_authors = db.all_authors()
+        all_authors.sort(key=lambda x : sort_key(x[1]))
+        for i in all_authors:
+            id, name = i
+            name = name.strip().replace('|', ',')
+            self.authors_box.addItem(name)
+        self.authors_box.setEditText('')
+        self.authors_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+        self.authors_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+        all_series = db.all_series()
+        all_series.sort(key=lambda x : sort_key(x[1]))
+        for i in all_series:
+            id, name = i
+            self.series_box.addItem(name)
+        self.series_box.setEditText('')
+        self.series_box.completer().setCompletionMode(QCompleter.PopupCompletion)
+        self.series_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive)
+
+        all_tags = db.all_tags()
+        self.tags_box.update_tags_cache(all_tags)
+
         self.box_last_values = copy.deepcopy(box_values)
         if self.box_last_values:
             for k,v in self.box_last_values.items():
@@ -121,26 +143,34 @@ class SearchDialog(QDialog, Ui_Dialog):
             return tok
 
     def box_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+
         ans = []
         self.box_last_values = {}
         title = unicode(self.title_box.text()).strip()
         self.box_last_values['title_box'] = title
         if title:
-            ans.append('title:"' + title + '"')
+            ans.append('title:"' + self.mc + title + '"')
         author = unicode(self.authors_box.text()).strip()
         self.box_last_values['authors_box'] = author
         if author:
-            ans.append('author:"' + author + '"')
+            ans.append('author:"' + self.mc + author + '"')
         series = unicode(self.series_box.text()).strip()
         self.box_last_values['series_box'] = series
         if series:
-            ans.append('series:"' + series + '"')
-        self.mc = '='
+            ans.append('series:"' + self.mc + series + '"')
+
         tags = unicode(self.tags_box.text())
         self.box_last_values['tags_box'] = tags
-        tags = self.tokens(tags)
+        tags = [t.strip() for t in tags.split(',') if t.strip()]
         if tags:
-            tags = ['tags:' + t for t in tags]
+            tags = ['tags:"=' + t + '"' for t in tags]
             ans.append('(' + ' or '.join(tags) + ')')
         general = unicode(self.general_box.text())
         self.box_last_values['general_box'] = general
diff --git a/src/calibre/gui2/dialogs/search.ui b/src/calibre/gui2/dialogs/search.ui
index 7bb4c15363..6848a45506 100644
--- a/src/calibre/gui2/dialogs/search.ui
+++ b/src/calibre/gui2/dialogs/search.ui
@@ -21,7 +21,7 @@
    <item row="0" column="0">
     <widget class="QLabel" name="label_5">
      <property name="text">
-      <string>What kind of match to use:</string>
+      <string>&amp;What kind of match to use:</string>
      </property>
      <property name="buddy">
       <cstring>matchkind</cstring>
@@ -228,7 +228,7 @@
         </widget>
        </item>
        <item row="1" column="1">
-        <widget class="QLineEdit" name="title_box">
+        <widget class="EnLineEdit" name="title_box">
          <property name="toolTip">
           <string>Enter the title.</string>
          </property>
@@ -265,21 +265,21 @@
         </widget>
        </item>
        <item row="2" column="1">
-        <widget class="QLineEdit" name="authors_box">
+        <widget class="EnComboBox" name="authors_box">
          <property name="toolTip">
           <string>Enter an author's name. Only one author can be used.</string>
          </property>
         </widget>
        </item>
        <item row="3" column="1">
-        <widget class="QLineEdit" name="series_box">
+        <widget class="EnComboBox" name="series_box">
          <property name="toolTip">
           <string>Enter a series name, without an index. Only one series name can be used.</string>
          </property>
         </widget>
        </item>
        <item row="4" column="1">
-        <widget class="QLineEdit" name="tags_box">
+        <widget class="TagsLineEdit" name="tags_box">
          <property name="toolTip">
           <string>Enter tags separated by spaces</string>
          </property>
@@ -348,6 +348,23 @@
    </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>EnLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>EnComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
+  <customwidget>
+   <class>TagsLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+ </customwidgets>
  <tabstops>
   <tabstop>all</tabstop>
   <tabstop>phrase</tabstop>
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 07ea407460..ba218c3ecc 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -133,7 +133,15 @@ class CustomColumns(object):
 
         def adapt_bool(x, d):
             if isinstance(x, (str, unicode, bytes)):
-                x = bool(int(x))
+                x = x.lower()
+                if x == 'true':
+                    x = True
+                elif x == 'false':
+                    x = False
+                elif x == 'none':
+                    x = None
+                else:
+                    x = bool(int(x))
             return x
 
         def adapt_enum(x, d):
@@ -142,9 +150,17 @@ class CustomColumns(object):
                 v = None
             return v
 
+        def adapt_number(x, d):
+            if isinstance(x, (str, unicode, bytes)):
+                if x.lower() == 'none':
+                    return None
+            if d['datatype'] == 'int':
+                return int(x)
+            return float(x)
+
         self.custom_data_adapters = {
-                'float': lambda x,d : x if x is None else float(x),
-                'int':   lambda x,d : x if x is None else int(x),
+                'float': adapt_number,
+                'int':   adapt_number,
                 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),

From 7e7f0954ce61c319b1223e8fe1baee53dfbb58ff Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 18:51:35 +0000
Subject: [PATCH 22/33] New implementation of CacheRow

---
 src/calibre/library/caches.py | 54 ++++++++++++++---------------------
 1 file changed, 21 insertions(+), 33 deletions(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 3a61a8fd5d..ada1ee0a77 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -132,47 +132,35 @@ def _match(query, value, matchkind):
             pass
     return False
 
-class CacheRow(object):
+class CacheRow(list):
 
     def __init__(self, db, composites, val):
         self.db = db
-        self.composites = composites
-        self._mydata = val
+        self._composites = composites
+        list.__init__(self, val)
         self._must_do = len(composites) > 0
 
     def __getitem__(self, col):
-        rec = self._mydata
-        if self._must_do and col in self.composites:
-            self._must_do = False
-            mi = self.db.get_metadata(rec[0], index_is_id=True)
-            for c in self.composites:
-                rec[c] = mi.get(self.composites[c])
-        return rec[col]
+        if self._must_do:
+            is_comp = False
+            if isinstance(col, slice):
+                for c in range(col.start, col.stop):
+                    if c in self._composites:
+                        is_comp = True
+                        break
+            elif col in self._composites:
+                is_comp = True
+            if is_comp:
+                id = list.__getitem__(self, 0)
+                self._must_do = False
+                mi = self.db.get_metadata(id, index_is_id=True)
+                for c in self._composites:
+                    self[c] =  mi.get(self._composites[c])
+        return list.__getitem__(self, col)
 
-    def __setitem__ (self, col, val):
-        self._mydata[col] = val
+    def __getslice__(self, i, j):
+        return self.__getitem__(slice(i, j))
 
-    def append(self, val):
-        self._mydata.append(val)
-
-    def get(self, col, default):
-        try:
-            return self.__getitem__(col)
-        except:
-            return default
-
-    def __len__(self):
-        return len(self._mydata)
-
-    def __iter__(self):
-        for v in self._mydata:
-            yield v
-
-    def __str__(self):
-        return self.__unicode__()
-
-    def __unicode__(self):
-        return unicode(self._mydata)
 
 class ResultCache(SearchQueryParser): # {{{
 

From e1bbefba8390859b0a7a50d24633eb4c0245ecb0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 12:15:37 -0700
Subject: [PATCH 23/33] Fix #4442 (Make input character encoding selection more
 user-friendly)

---
 src/calibre/gui2/convert/look_and_feel.ui | 17 ++++++++++++---
 src/calibre/gui2/widgets.py               | 25 +++++++++++++++++++++++
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui
index 367233e2c0..0edc324dc5 100644
--- a/src/calibre/gui2/convert/look_and_feel.ui
+++ b/src/calibre/gui2/convert/look_and_feel.ui
@@ -127,9 +127,6 @@
      </property>
     </widget>
    </item>
-   <item row="5" column="1" colspan="3">
-    <widget class="QLineEdit" name="opt_input_encoding"/>
-   </item>
    <item row="6" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
      <property name="text">
@@ -244,8 +241,22 @@
      </property>
     </widget>
    </item>
+   <item row="5" column="1" colspan="3">
+    <widget class="EncodingComboBox" name="opt_input_encoding">
+     <property name="editable">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
+ <customwidgets>
+  <customwidget>
+   <class>EncodingComboBox</class>
+   <extends>QComboBox</extends>
+   <header>widgets.h</header>
+  </customwidget>
+ </customwidgets>
  <resources>
   <include location="../../../../resources/images.qrc"/>
   <include location="../../../../resources/images.qrc"/>
diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py
index bc3c23876f..d87bb45f7a 100644
--- a/src/calibre/gui2/widgets.py
+++ b/src/calibre/gui2/widgets.py
@@ -616,6 +616,31 @@ class ComboBoxWithHelp(QComboBox):
         QComboBox.hidePopup(self)
         self.set_state()
 
+
+class EncodingComboBox(QComboBox):
+    '''
+    A combobox that holds text encodings support
+    by Python. This is only populated with the most
+    common and standard encodings. There is no good
+    way to programatically list all supported encodings
+    using encodings.aliases.aliases.keys(). It
+    will not work.
+    '''
+
+    ENCODINGS = ['', 'cp1252', 'latin1', 'utf-8', '', 'ascii', 'big5', 'cp1250', 'cp1251', 'cp1253',
+        'cp1254', 'cp1255', 'cp1256', 'euc_jp', 'euc_kr', 'gb2312', 'gb18030',
+        'hz', 'iso2022_jp', 'iso2022_kr', 'iso8859_5', 'shift_jis',
+    ]
+
+    def __init__(self, parent=None):
+        QComboBox.__init__(self, parent)
+        self.setEditable(True)
+        self.setLineEdit(EnLineEdit(self))
+
+        for item in self.ENCODINGS:
+            self.addItem(item)
+
+
 class PythonHighlighter(QSyntaxHighlighter):
 
     Rules = []

From cbd880e8f736fe3ee20c3fea62a1895a8848194f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:17:01 +0000
Subject: [PATCH 24/33] Take out sorting change

---
 src/calibre/gui2/library/models.py    |  4 ++--
 src/calibre/library/custom_columns.py | 31 ++++++---------------------
 src/calibre/library/database2.py      |  2 +-
 src/calibre/library/sqlite.py         | 23 +-------------------
 4 files changed, 11 insertions(+), 49 deletions(-)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 6a48aef9be..49cb1ce182 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -526,7 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def tags(r, idx=-1):
             tags = self.db.data[r][idx]
             if tags:
-                return QVariant(', '.join(tags.split(',')))
+                return QVariant(', '.join(sorted(tags.split(','), key=sort_key)))
             return None
 
         def series_type(r, idx=-1, siix=-1):
@@ -577,7 +577,7 @@ class BooksModel(QAbstractTableModel): # {{{
         def text_type(r, mult=False, idx=-1):
             text = self.db.data[r][idx]
             if text and mult:
-                return QVariant(', '.join(text.split('|')))
+                return QVariant(', '.join(sorted(text.split('|'),key=sort_key)))
             return QVariant(text)
 
         def number_type(r, idx=-1):
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index ccdd55021d..07ea407460 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -14,7 +14,6 @@ from calibre.constants import preferred_encoding
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import parse_date
 from calibre.utils.config import tweaks
-from calibre.utils.icu import sort_key
 
 class CustomColumns(object):
 
@@ -134,15 +133,7 @@ class CustomColumns(object):
 
         def adapt_bool(x, d):
             if isinstance(x, (str, unicode, bytes)):
-                x = x.lower()
-                if x == 'true':
-                    x = True
-                elif x == 'false':
-                    x = False
-                elif x == 'none':
-                    x = None
-                else:
-                    x = bool(int(x))
+                x = bool(int(x))
             return x
 
         def adapt_enum(x, d):
@@ -151,17 +142,9 @@ class CustomColumns(object):
                 v = None
             return v
 
-        def adapt_number(x, d):
-            if isinstance(x, (str, unicode, bytes)):
-                if x.lower() == 'none':
-                    return None
-            if d['datatype'] == 'int':
-                return int(x)
-            return float(x)
-
         self.custom_data_adapters = {
-                'float': adapt_number,
-                'int':   adapt_number,
+                'float': lambda x,d : x if x is None else float(x),
+                'int':   lambda x,d : x if x is None else int(x),
                 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))),
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
@@ -198,8 +181,8 @@ class CustomColumns(object):
         ans = row[self.FIELD_MAP[data['num']]]
         if data['is_multiple'] and data['datatype'] == 'text':
             ans = ans.split('|') if ans else []
-            if data['display'].get('sort_alpha', True):
-                ans.sort(key=sort_key)
+            if data['display'].get('sort_alpha', False):
+                ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
 
     def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
@@ -551,8 +534,8 @@ class CustomColumns(object):
             if data['normalized']:
                 query = '%s.value'
                 if data['is_multiple']:
-                    query = 'cc_sortconcat(%s.value)'
-                    if not display.get('sort_alpha', True):
+                    query = 'group_concat(%s.value, "|")'
+                    if not display.get('sort_alpha', False):
                         query = 'sort_concat(link.id, %s.value)'
                 line = '''(SELECT {query} FROM {lt} AS link INNER JOIN
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 0b1c6a6cfb..611aa1cc89 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -242,7 +242,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'timestamp',
              '(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size',
             ('rating', 'ratings', 'rating', 'ratings.rating'),
-            ('tags', 'tags', 'tag', 'tags_sortconcat(name)'),
+            ('tags', 'tags', 'tag', 'group_concat(name)'),
              '(SELECT text FROM comments WHERE book=books.id) comments',
             ('series', 'series', 'series', 'name'),
             ('publisher', 'publishers', 'publisher', 'name'),
diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py
index 0c3ae487ea..0458ada27b 100644
--- a/src/calibre/library/sqlite.py
+++ b/src/calibre/library/sqlite.py
@@ -19,7 +19,7 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.date import parse_date, isoformat
 from calibre import isbytestring, force_unicode
 from calibre.constants import iswindows, DEBUG
-from calibre.utils.icu import strcmp, sort_key
+from calibre.utils.icu import strcmp
 
 global_lock = RLock()
 
@@ -69,25 +69,6 @@ class Concatenate(object):
             return None
         return self.sep.join(self.ans)
 
-class TagsSortConcatenate(object):
-    '''Sorted string concatenation aggregator for sqlite'''
-    def __init__(self, sep=','):
-        self.sep = sep
-        self.ans = []
-
-    def step(self, value):
-        if value is not None:
-            self.ans.append(value)
-
-    def finalize(self):
-        if not self.ans:
-            return None
-        return self.sep.join(sorted(self.ans, key=sort_key))
-
-class CcSortConcatenate(TagsSortConcatenate):
-    def __init__(self):
-        TagsSortConcatenate.__init__(self, sep='|')
-
 class SortedConcatenate(object):
     '''String concatenation aggregator for sqlite, sorted by supplied index'''
     sep = ','
@@ -174,8 +155,6 @@ class DBThread(Thread):
         c_ext_loaded = load_c_extensions(self.conn)
         self.conn.row_factory = sqlite.Row if self.row_factory else  lambda cursor, row : list(row)
         self.conn.create_aggregate('concat', 1, Concatenate)
-        self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate)
-        self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate)
         if not c_ext_loaded:
             self.conn.create_aggregate('sortconcat', 2, SortedConcatenate)
             self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate)

From 9ee7dc27e85f1d6e948210818a4b68082bf5792d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:21:17 +0000
Subject: [PATCH 25/33] Do slice stepping correctly

---
 src/calibre/library/caches.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index ada1ee0a77..2596b494bf 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -144,7 +144,9 @@ class CacheRow(list):
         if self._must_do:
             is_comp = False
             if isinstance(col, slice):
-                for c in range(col.start, col.stop):
+                start = 0 if col.start is None else col.start
+                step = 1 if col.stop is None else col.stop
+                for c in range(start, col.stop, step):
                     if c in self._composites:
                         is_comp = True
                         break

From 5adaf263e4c29cc3810a198f44777d90b5999ad5 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 5 Jan 2011 19:52:55 +0000
Subject: [PATCH 26/33] Ensure that multiples are sorted in alpha order before
 column sort and during template processing

---
 src/calibre/ebooks/metadata/book/base.py |  7 ++++---
 src/calibre/library/caches.py            | 15 ++++++---------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 77df6b00c2..17f2c6705c 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -16,6 +16,7 @@ from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS
 from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import isoformat, format_date
+from calibre.utils.icu import sort_key
 from calibre.utils.formatter import TemplateFormatter
 
 
@@ -490,7 +491,7 @@ class Metadata(object):
         return authors_to_string(self.authors)
 
     def format_tags(self):
-        return u', '.join([unicode(t) for t in self.tags])
+        return u', '.join([unicode(t) for t in sorted(self.tags, key=sort_key)])
 
     def format_rating(self):
         return unicode(self.rating)
@@ -530,7 +531,7 @@ class Metadata(object):
             orig_res = res
             datatype = cmeta['datatype']
             if datatype == 'text' and cmeta['is_multiple']:
-                res = u', '.join(res)
+                res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 if self.get_extra(key) is not None:
                     res = res + \
@@ -560,7 +561,7 @@ class Metadata(object):
             elif key == 'series_index':
                 res = self.format_series_index(res)
             elif datatype == 'text' and fmeta['is_multiple']:
-                res = u', '.join(res)
+                res = u', '.join(sorted(res, key=sort_key))
             elif datatype == 'series' and series_with_index:
                 res = res + ' [%s]'%self.format_series_index()
             elif datatype == 'datetime':
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 2596b494bf..980c9f1fa9 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -691,13 +691,7 @@ class ResultCache(SearchQueryParser): # {{{
             fields = [('timestamp', False)]
 
         keyg = SortKeyGenerator(fields, self.field_metadata, self._data)
-        # For efficiency, the key generator returns a plain value if only one
-        # field is in the sort field list. Because the normal cmp function will
-        # always assume asc, we must deal with asc/desc here.
-        if len(fields) == 1:
-            self._map.sort(key=keyg, reverse=not fields[0][1])
-        else:
-            self._map.sort(key=keyg)
+        self._map.sort(key=keyg)
 
         tmap = list(itertools.repeat(False, len(self._data)))
         for x in self._map_filtered:
@@ -730,8 +724,6 @@ class SortKeyGenerator(object):
 
     def __call__(self, record):
         values = tuple(self.itervals(self.data[record]))
-        if len(values) == 1:
-            return values[0]
         return SortKey(self.orders, values)
 
     def itervals(self, record):
@@ -754,6 +746,11 @@ class SortKeyGenerator(object):
                     val = (self.string_sort_key(val), sidx)
 
             elif dt in ('text', 'comments', 'composite', 'enumeration'):
+                if val:
+                    sep = fm['is_multiple']
+                    if sep:
+                        val = sep.join(sorted(val.split(sep),
+                                              key=self.string_sort_key))
                 val = self.string_sort_key(val)
 
             elif dt == 'bool':

From 59907f91c7fc684013d0a67326de9b4bd9276a16 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 14:32:39 -0700
Subject: [PATCH 27/33] When downloading metadata from isbndb.com, download a
 maximum of 30 results rather than 1000

---
 src/calibre/ebooks/metadata/isbndb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py
index 9169227326..1c5f706593 100644
--- a/src/calibre/ebooks/metadata/isbndb.py
+++ b/src/calibre/ebooks/metadata/isbndb.py
@@ -17,10 +17,10 @@ BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&res
 class ISBNDBError(Exception):
     pass
 
-def fetch_metadata(url, max=100, timeout=5.):
+def fetch_metadata(url, max=3, timeout=5.):
     books = []
     page_number = 1
-    total_results = sys.maxint
+    total_results = 31
     br = browser()
     while len(books) < total_results and max > 0:
         try:

From 8a70e2dc3a11e075ff83ccbc4213a2b57ff37d0c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 14:39:46 -0700
Subject: [PATCH 28/33] ...

---
 src/calibre/ebooks/metadata/xisbn.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/metadata/xisbn.py b/src/calibre/ebooks/metadata/xisbn.py
index 21ea0ee79f..2ee74396c7 100644
--- a/src/calibre/ebooks/metadata/xisbn.py
+++ b/src/calibre/ebooks/metadata/xisbn.py
@@ -18,7 +18,6 @@ class xISBN(object):
         self._data = []
         self._map = {}
 
-        self.br = browser()
         self.isbn_pat = re.compile(r'[^0-9X]', re.IGNORECASE)
 
     def purify(self, isbn):
@@ -26,7 +25,7 @@ class xISBN(object):
 
     def fetch_data(self, isbn):
         url = self.QUERY%isbn
-        data = self.br.open_novisit(url).read()
+        data = browser().open_novisit(url).read()
         data = json.loads(data)
         if data.get('stat', None) != 'ok':
             return []

From f42cda6573e2380e8911e1a7ddee8fb976547abf Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 14:53:48 -0700
Subject: [PATCH 29/33] ...

---
 src/calibre/ebooks/oeb/transforms/cover.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/oeb/transforms/cover.py b/src/calibre/ebooks/oeb/transforms/cover.py
index a3cf9e9f83..0d82e9ad73 100644
--- a/src/calibre/ebooks/oeb/transforms/cover.py
+++ b/src/calibre/ebooks/oeb/transforms/cover.py
@@ -103,7 +103,7 @@ class CoverManager(object):
             from calibre.ebooks import calibre_cover
             img_data = calibre_cover(title, authors_to_string(authors),
                     series_string=series_string)
-            id, href = self.oeb.manifest.generate('cover_image',
+            id, href = self.oeb.manifest.generate('cover',
                     'cover_image.jpg')
             item = self.oeb.manifest.add(id, href, guess_type('t.jpg')[0],
                         data=img_data)

From 847422e9b0eadaf23327167433930114103cd3e2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 14:54:14 -0700
Subject: [PATCH 30/33] Fix #8202 (Wrong Folder on device dr900)

---
 src/calibre/devices/misc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py
index 2a0fdf6433..ecd12ac61d 100644
--- a/src/calibre/devices/misc.py
+++ b/src/calibre/devices/misc.py
@@ -259,7 +259,7 @@ class EEEREADER(USBMS):
     PRODUCT_ID  = [0x178f]
     BCD         = [0x0319]
 
-    EBOOK_DIR_MAIN = 'Books'
+    EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'Book'
 
     VENDOR_NAME = 'LINUX'
     WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET'

From 51314ce5d8b6af02751b714606017a0f28b36bc7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 15:26:21 -0700
Subject: [PATCH 31/33] Content server: When serving OPDS feeds handle html
 descriptions taht have namespaced attributes. Fixes #7938 (Stanza shows some
 authors as "catalog is empty")

---
 src/calibre/library/server/opds.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py
index fd8c50c594..ab0853add9 100644
--- a/src/calibre/library/server/opds.py
+++ b/src/calibre/library/server/opds.py
@@ -101,7 +101,19 @@ def html_to_lxml(raw):
     root = html.fragment_fromstring(raw)
     root.set('xmlns', "http://www.w3.org/1999/xhtml")
     raw = etree.tostring(root, encoding=None)
-    return etree.fromstring(raw)
+    try:
+        return etree.fromstring(raw)
+    except:
+        for x in root.iterdescendants():
+            remove = []
+            for attr in x.attrib:
+                if ':' in attr:
+                    remove.append(attr)
+            for a in remove:
+                del x.attrib[a]
+        raw = etree.tostring(root, encoding=None)
+        return etree.fromstring(raw)
+
 
 def CATALOG_ENTRY(item, item_kind, base_href, version, updated,
                   ignore_count=False, add_kind=False):

From 67c9bec212c88406e04bf31855ee28744c735525 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 19:04:40 -0700
Subject: [PATCH 32/33] Fix #8176 (Expose configuration dialog for a plugin via
 API)

---
 src/calibre/customize/__init__.py       | 78 +++++++++++++++++++++++++
 src/calibre/gui2/preferences/plugins.py | 49 +---------------
 2 files changed, 81 insertions(+), 46 deletions(-)

diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py
index 770d405203..a00a2cd5e2 100644
--- a/src/calibre/customize/__init__.py
+++ b/src/calibre/customize/__init__.py
@@ -80,6 +80,84 @@ class Plugin(object): # {{{
         '''
         pass
 
+    def config_widget(self):
+        '''
+        Implement this method and :meth:`save_settings` in your plugin to
+        use a custom configuration dialog, rather then relying on the simple
+        string based default customization.
+
+        This method, if implemented, must return a QWidget. The widget can have
+        an optional method validate() that takes no arguments and is called
+        immediately after the user clicks OK. Changes are applied if and only
+        if the method returns True.
+        '''
+        raise NotImplementedError()
+
+    def save_settings(self, config_widget):
+        '''
+        Save the settings specified by the user with config_widget.
+
+        :param config_widget: The widget returned by :meth:`config_widget`.
+
+        '''
+        raise NotImplementedError()
+
+    def do_user_config(self, parent=None):
+        '''
+        This method shows a configuration dialog for this plugin. It returns
+        True if the user clicks OK, False otherwise. The changes are
+        automatically applied.
+        '''
+        from PyQt4.Qt import QDialog, QDialogButtonBox, QVBoxLayout, \
+                QLabel, Qt, QLineEdit
+        config_dialog = QDialog(parent)
+        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        v = QVBoxLayout(config_dialog)
+
+        button_box.accepted.connect(config_dialog.accept)
+        button_box.rejected.connect(config_dialog.reject)
+        config_dialog.setWindowTitle(_('Customize') + ' ' + self.name)
+        try:
+            config_widget = self.config_widget()
+        except NotImplementedError:
+            config_widget = None
+
+        if config_widget is not None:
+            v.addWidget(config_widget)
+            v.addWidget(button_box)
+            config_dialog.exec_()
+
+            if config_dialog.result() == QDialog.Accepted:
+                if hasattr(config_widget, 'validate'):
+                    if config_widget.validate():
+                        self.save_settings(config_widget)
+                else:
+                    self.save_settings(config_widget)
+        else:
+            from calibre.customize.ui import plugin_customization, \
+                customize_plugin
+            help_text = self.customization_help(gui=True)
+            help_text = QLabel(help_text, config_dialog)
+            help_text.setWordWrap(True)
+            help_text.setTextInteractionFlags(Qt.LinksAccessibleByMouse
+                    | Qt.LinksAccessibleByKeyboard)
+            help_text.setOpenExternalLinks(True)
+            v.addWidget(help_text)
+            sc = plugin_customization(self)
+            if not sc:
+                sc = ''
+            sc = sc.strip()
+            sc = QLineEdit(sc, config_dialog)
+            v.addWidget(sc)
+            v.addWidget(button_box)
+            config_dialog.exec_()
+
+            if config_dialog.result() == QDialog.Accepted:
+                sc = unicode(sc.text()).strip()
+                customize_plugin(self, sc)
+
+        return config_dialog.result()
+
     def load_resources(self, names):
         '''
         If this plugin comes in a ZIP file (user added plugin), this method
diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py
index 3d714e388e..2fe2b3bf01 100644
--- a/src/calibre/gui2/preferences/plugins.py
+++ b/src/calibre/gui2/preferences/plugins.py
@@ -8,13 +8,12 @@ __docformat__ = 'restructuredtext en'
 import textwrap, os
 
 from PyQt4.Qt import Qt, QModelIndex, QAbstractItemModel, QVariant, QIcon, \
-        QBrush, QDialog, QDialogButtonBox, QVBoxLayout, QLabel, QLineEdit
+        QBrush
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.plugins_ui import Ui_Form
 from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \
-                                 disable_plugin, customize_plugin, \
-                                 plugin_customization, add_plugin, \
+                                 disable_plugin, plugin_customization, add_plugin, \
                                  remove_plugin
 from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files
 
@@ -189,49 +188,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                         _('Plugin: %s does not need customization')%plugin.name).exec_()
                     return
                 self.changed_signal.emit()
-
-                config_dialog = QDialog(self)
-                button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-                v = QVBoxLayout(config_dialog)
-
-                button_box.accepted.connect(config_dialog.accept)
-                button_box.rejected.connect(config_dialog.reject)
-                config_dialog.setWindowTitle(_('Customize') + ' ' + plugin.name)
-
-                if hasattr(plugin, 'config_widget'):
-                    config_widget = plugin.config_widget()
-                    v.addWidget(config_widget)
-                    v.addWidget(button_box)
-                    config_dialog.exec_()
-
-                    if config_dialog.result() == QDialog.Accepted:
-                        if hasattr(config_widget, 'validate'):
-                            if config_widget.validate():
-                                plugin.save_settings(config_widget)
-                        else:
-                            plugin.save_settings(config_widget)
-                        self._plugin_model.refresh_plugin(plugin)
-                else:
-                    help_text = plugin.customization_help(gui=True)
-                    help_text = QLabel(help_text, config_dialog)
-                    help_text.setWordWrap(True)
-                    help_text.setTextInteractionFlags(Qt.LinksAccessibleByMouse
-                            | Qt.LinksAccessibleByKeyboard)
-                    help_text.setOpenExternalLinks(True)
-                    v.addWidget(help_text)
-                    sc = plugin_customization(plugin)
-                    if not sc:
-                        sc = ''
-                    sc = sc.strip()
-                    sc = QLineEdit(sc, config_dialog)
-                    v.addWidget(sc)
-                    v.addWidget(button_box)
-                    config_dialog.exec_()
-
-                    if config_dialog.result() == QDialog.Accepted:
-                        sc = unicode(sc.text()).strip()
-                        customize_plugin(plugin, sc)
-
+                if plugin.do_user_config():
                     self._plugin_model.refresh_plugin(plugin)
             elif op == 'remove':
                 if remove_plugin(plugin):

From 5c6a17b25b04297692e953f20c09d3da62c0a26b Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 5 Jan 2011 19:11:59 -0700
Subject: [PATCH 33/33] Fix #8174 (Persist plugin configuration dialog
 size/position)

---
 src/calibre/customize/__init__.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py
index a00a2cd5e2..13e1f20a2d 100644
--- a/src/calibre/customize/__init__.py
+++ b/src/calibre/customize/__init__.py
@@ -110,10 +110,21 @@ class Plugin(object): # {{{
         '''
         from PyQt4.Qt import QDialog, QDialogButtonBox, QVBoxLayout, \
                 QLabel, Qt, QLineEdit
+        from calibre.gui2 import gprefs
+
+        prefname = 'plugin config dialog:'+self.type + ':' + self.name
+        geom = gprefs.get(prefname, None)
+
         config_dialog = QDialog(parent)
         button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
         v = QVBoxLayout(config_dialog)
 
+        def size_dialog():
+            if geom is None:
+                config_dialog.resize(config_dialog.sizeHint())
+            else:
+                config_dialog.restoreGeometry(geom)
+
         button_box.accepted.connect(config_dialog.accept)
         button_box.rejected.connect(config_dialog.reject)
         config_dialog.setWindowTitle(_('Customize') + ' ' + self.name)
@@ -125,6 +136,7 @@ class Plugin(object): # {{{
         if config_widget is not None:
             v.addWidget(config_widget)
             v.addWidget(button_box)
+            size_dialog()
             config_dialog.exec_()
 
             if config_dialog.result() == QDialog.Accepted:
@@ -150,12 +162,16 @@ class Plugin(object): # {{{
             sc = QLineEdit(sc, config_dialog)
             v.addWidget(sc)
             v.addWidget(button_box)
+            size_dialog()
             config_dialog.exec_()
 
             if config_dialog.result() == QDialog.Accepted:
                 sc = unicode(sc.text()).strip()
                 customize_plugin(self, sc)
 
+        geom = bytearray(config_dialog.saveGeometry())
+        gprefs[prefname] = geom
+
         return config_dialog.result()
 
     def load_resources(self, names):