From 6b80d5d0311ffb2f5d20f27aa21b2351efdeae6a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 20 Jun 2010 11:25:24 +0100
Subject: [PATCH 01/31] Series type: attempt using 1 column with a virtual
 second column

---
 .../dialogs/config/create_custom_column.py    | 13 ++--
 src/calibre/gui2/library/models.py            | 26 ++++++--
 src/calibre/library/caches.py                 | 42 +++++++-----
 src/calibre/library/custom_columns.py         | 66 +++++++++++++++----
 src/calibre/library/database2.py              |  5 ++
 src/calibre/library/field_metadata.py         |  4 +-
 6 files changed, 117 insertions(+), 39 deletions(-)

diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.py b/src/calibre/gui2/dialogs/config/create_custom_column.py
index a66b7b6642..2aae567b1c 100644
--- a/src/calibre/gui2/dialogs/config/create_custom_column.py
+++ b/src/calibre/gui2/dialogs/config/create_custom_column.py
@@ -24,16 +24,19 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     2:{'datatype':'comments',
                         'text':_('Long text, like comments, not shown in the tag browser'),
                         'is_multiple':False},
-                    3:{'datatype':'datetime',
+                    3:{'datatype':'series',
+                        'text':_('Text column for keeping series-like information'),
+                        'is_multiple':False},
+                    4:{'datatype':'datetime',
                         'text':_('Date'), 'is_multiple':False},
-                    4:{'datatype':'float',
+                    5:{'datatype':'float',
                         'text':_('Floating point numbers'), 'is_multiple':False},
-                    5:{'datatype':'int',
+                    6:{'datatype':'int',
                         'text':_('Integers'), 'is_multiple':False},
-                    6:{'datatype':'rating',
+                    7:{'datatype':'rating',
                         'text':_('Ratings, shown with stars'),
                         'is_multiple':False},
-                    7:{'datatype':'bool',
+                    8:{'datatype':'bool',
                         'text':_('Yes/No'), 'is_multiple':False},
                 }
 
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index dcc338dbdc..435b5c4c07 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -520,7 +520,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(', '.join(sorted(tags.split(','))))
             return None
 
-        def series(r, idx=-1, siix=-1):
+        def series_type(r, idx=-1, siix=-1):
             series = self.db.data[r][idx]
             if series:
                 idx = fmt_sidx(self.db.data[r][siix])
@@ -591,7 +591,7 @@ class BooksModel(QAbstractTableModel): # {{{
                                 idx=self.db.field_metadata['publisher']['rec_index'], mult=False),
                    'tags'     : functools.partial(tags,
                                 idx=self.db.field_metadata['tags']['rec_index']),
-                   'series'   : functools.partial(series,
+                   'series'   : functools.partial(series_type,
                                 idx=self.db.field_metadata['series']['rec_index'],
                                 siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
@@ -620,6 +620,9 @@ class BooksModel(QAbstractTableModel): # {{{
                                             bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
+            elif datatype == 'series':
+                self.dc[col] = functools.partial(series_type, idx=idx,
+                    siix=self.db.field_metadata.cc_series_index_column_for(col))
             else:
                 print 'What type is this?', col, datatype
         # build a index column to data converter map, to remove the string lookup in the data loop
@@ -681,6 +684,8 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_custom_column_data(self, row, colhead, value):
         typ = self.custom_columns[colhead]['datatype']
+        label=self.db.field_metadata.key_to_label(colhead)
+        s_index = None
         if typ in ('text', 'comments'):
             val = unicode(value.toString()).strip()
             val = val if val else None
@@ -702,9 +707,20 @@ class BooksModel(QAbstractTableModel): # {{{
                 if not val.isValid():
                     return False
                 val = qt_to_dt(val, as_utc=False)
-        self.db.set_custom(self.db.id(row), val,
-                           label=self.db.field_metadata.key_to_label(colhead),
-                           num=None, append=False, notify=True)
+        elif typ == 'series':
+            val = unicode(value.toString()).strip()
+            pat = re.compile(r'\[([.0-9]+)\]')
+            match = pat.search(val)
+            if match is not None:
+                val = pat.sub('', val).strip()
+                s_index = float(match.group(1))
+            elif val:
+                if tweaks['series_index_auto_increment'] == 'next':
+                    s_index = self.db.get_next_cc_series_num_for(val, label=label)
+                else:
+                    s_index = 1.0
+        self.db.set_custom(self.db.id(row), val, extra=s_index,
+                           label=label, num=None, append=False, notify=True)
         return True
 
     def setData(self, index, value, role):
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 57f9d0baaf..d4afaabcdc 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -401,7 +401,8 @@ class ResultCache(SearchQueryParser):
             for x in self.field_metadata:
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
-                    if self.field_metadata[x]['datatype'] not in ['text', 'comments']:
+                    if self.field_metadata[x]['datatype'] not in \
+                                                ['text', 'comments', 'series']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
                     is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@@ -580,16 +581,18 @@ class ResultCache(SearchQueryParser):
             self.sort(field, ascending)
         self._map_filtered = list(self._map)
 
-    def seriescmp(self, x, y):
-        sidx = self.FIELD_MAP['series']
+    def seriescmp(self, sidx, siidx, x, y, library_order=None):
         try:
-            ans = cmp(title_sort(self._data[x][sidx].lower()),
-                    title_sort(self._data[y][sidx].lower()))
+            if library_order:
+                ans = cmp(title_sort(self._data[x][sidx].lower()),
+                                title_sort(self._data[y][sidx].lower()))
+            else:
+                ans = cmp(self._data[x][sidx].lower(),
+                                                self._data[y][sidx].lower())
         except AttributeError: # Some entries may be None
             ans = cmp(self._data[x][sidx], self._data[y][sidx])
         if ans != 0: return ans
-        sidx = self.FIELD_MAP['series_index']
-        return cmp(self._data[x][sidx], self._data[y][sidx])
+        return cmp(self._data[x][siidx], self._data[y][siidx])
 
     def cmp(self, loc, x, y, asstr=True, subsort=False):
         try:
@@ -618,18 +621,27 @@ class ResultCache(SearchQueryParser):
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
         if self.field_metadata[field]['is_custom']:
-            as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-            field = self.field_metadata[field]['colnum']
+            if self.field_metadata[field]['datatype'] == 'series':
+                fcmp = functools.partial(self.seriescmp,
+                    self.field_metadata[field]['rec_index'],
+                    self.field_metadata.cc_series_index_column_for(field),
+                    library_order=tweaks['title_series_sorting'] == 'library_order')
+            else:
+                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
+                field = self.field_metadata[field]['colnum']
+                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                                     subsort=subsort, asstr=as_string)
+        elif field == 'series':
+            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
+                self.FIELD_MAP['series_index'],
+                library_order=tweaks['title_series_sorting'] == 'library_order')
+        else:
+            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                                     subsort=subsort, asstr=as_string)
 
         if self.first_sort:
             subsort = True
             self.first_sort = False
-        fcmp = self.seriescmp \
-                if field == 'series' and \
-                   tweaks['title_series_sorting'] == 'library_order' \
-                else \
-                   functools.partial(self.cmp, self.FIELD_MAP[field],
-                                     subsort=subsort, asstr=as_string)
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]
 
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index c0ba91e252..a5f9ddb4f0 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import json
 from functools import partial
+from math import floor
 
 from calibre import prints
 from calibre.constants import preferred_encoding
@@ -16,7 +17,7 @@ from calibre.utils.date import parse_date
 class CustomColumns(object):
 
     CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
-        'int', 'float', 'bool'])
+        'int', 'float', 'bool', 'series'])
 
     def custom_table_names(self, num):
         return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@@ -137,7 +138,8 @@ class CustomColumns(object):
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
                 'datetime' : adapt_datetime,
-                'text':adapt_text
+                'text':adapt_text,
+                'series':adapt_text
         }
 
         # Create Tag Browser categories for custom columns
@@ -220,6 +222,28 @@ class CustomColumns(object):
             self.conn.commit()
     # end convenience methods
 
+    def get_next_cc_series_num_for(self, series, label=None, num=None):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        if data['datatype'] != 'series':
+            return None
+        table, lt = self.custom_table_names(data['num'])
+        # get the id of the row containing the series string
+        series_id = self.conn.get('SELECT id from %s WHERE value=?'%table,
+                                                        (series,), all=False)
+        if series_id is None:
+            return 1.0
+        # get the label of the associated series number table
+        series_num = self.conn.get('''
+                SELECT MAX({lt}.s_index) FROM {lt}
+                WHERE {lt}.book IN (SELECT book FROM {lt} where value=?)
+                '''.format(lt=lt), (series_id,), all=False)
+        if series_num is None:
+            return 1.0
+        return floor(series_num+1)
+
     def all_custom(self, label=None, num=None):
         if label is not None:
             data = self.custom_column_label_map[label]
@@ -271,9 +295,8 @@ class CustomColumns(object):
             self.conn.commit()
         return changed
 
-
-
-    def set_custom(self, id_, val, label=None, num=None, append=False, notify=True):
+    def set_custom(self, id_, val, extra=None, label=None, num=None,
+                   append=False, notify=True):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:
@@ -317,10 +340,17 @@ class CustomColumns(object):
                             'INSERT INTO %s(value) VALUES(?)'%table, (x,)).lastrowid
                 if not self.conn.get(
                     'SELECT book FROM %s WHERE book=? AND value=?'%lt,
-                    (id_, xid), all=False):
-                    self.conn.execute(
-                        'INSERT INTO %s(book, value) VALUES (?,?)'%lt,
-                            (id_, xid))
+                                                        (id_, xid), all=False):
+                    if data['datatype'] == 'series':
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value, s_index)
+                               VALUES (?,?,?)'''%lt, (id_, xid, extra))
+                        self.data.set(id_, self.FIELD_MAP[data['num']]+1,
+                                      extra, row_is_id=True)
+                    else:
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value)
+                                VALUES (?,?)'''%lt, (id_, xid))
             self.conn.commit()
             nval = self.conn.get(
                     'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'],
@@ -370,6 +400,9 @@ class CustomColumns(object):
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}
                 '''.format(query=query%table, lt=lt, table=table, num=data['num'])
+                if data['datatype'] == 'series':
+                    line += ''',(SELECT s_index FROM {lt} WHERE {lt}.book=books.id)
+                        custom_index_{num}'''.format(lt=lt, num=data['num'])
             else:
                 line = '''
                 (SELECT value FROM {table} WHERE book=books.id) custom_{num}
@@ -393,7 +426,7 @@ class CustomColumns(object):
 
         if datatype in ('rating', 'int'):
             dt = 'INT'
-        elif datatype in ('text', 'comments'):
+        elif datatype in ('text', 'comments', 'series'):
             dt = 'TEXT'
         elif datatype in ('float',):
             dt = 'REAL'
@@ -404,6 +437,10 @@ class CustomColumns(object):
         collate = 'COLLATE NOCASE' if dt == 'TEXT' else ''
         table, lt = self.custom_table_names(num)
         if normalized:
+            if datatype == 'series':
+                s_index = 's_index REAL,'
+            else:
+                s_index = ''
             lines = [
                 '''\
                 CREATE TABLE %s(
@@ -419,8 +456,9 @@ class CustomColumns(object):
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     book INTEGER NOT NULL,
                     value INTEGER NOT NULL,
+                    %s
                     UNIQUE(book, value)
-                    );'''%lt,
+                    );'''%(lt, s_index),
 
                 'CREATE INDEX %s_aidx ON %s (value);'%(lt,lt),
                 'CREATE INDEX %s_bidx ON %s (book);'%(lt,lt),
@@ -467,7 +505,8 @@ class CustomColumns(object):
                           books_ratings_link as bl,
                           ratings as r
                      WHERE {lt}.value={table}.id and bl.book={lt}.book and
-                           r.id = bl.rating and r.rating <> 0) avg_rating
+                           r.id = bl.rating and r.rating <> 0) avg_rating,
+                    value as sort
                 FROM {table};
 
                 CREATE VIEW tag_browser_filtered_{table} AS SELECT
@@ -481,7 +520,8 @@ class CustomColumns(object):
                           ratings as r
                      WHERE {lt}.value={table}.id AND bl.book={lt}.book AND
                            r.id = bl.rating AND r.rating <> 0 AND
-                           books_list_filter(bl.book)) avg_rating
+                           books_list_filter(bl.book)) avg_rating,
+                    value as sort
                 FROM {table};
 
                 '''.format(lt=lt, table=table),
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index fe4aac12b5..7e4a879654 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -237,6 +237,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         self.custom_column_num_map[col]['label'],
                                         base,
                                         prefer_custom=True)
+            if self.custom_column_num_map[col]['datatype'] == 'series':
+                # account for the series index column. Field_metadata knows that
+                # the series index is one larger than the series. If you change
+                # it here, be sure to change it there as well.
+                self.FIELD_MAP[str(col)+'_s_index'] = base = base+1
 
         self.FIELD_MAP['cover'] = base+1
         self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py
index 8cb5c9bdad..5ccc17d1eb 100644
--- a/src/calibre/library/field_metadata.py
+++ b/src/calibre/library/field_metadata.py
@@ -81,7 +81,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
-                           'datatype':'text',
+                           'datatype':'series',
                            'is_multiple':None,
                            'kind':'field',
                            'name':_('Series'),
@@ -398,6 +398,8 @@ class FieldMetadata(dict):
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
 
+    def cc_series_index_column_for(self, key):
+        return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
         if label in self._tb_cats:

From 5544a5d2221f6841cc50e796f911aa4b0b4ece9a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 20 Jun 2010 19:21:53 +0100
Subject: [PATCH 02/31] Series custom columns seem to work

---
 src/calibre/gui2/custom_column_widgets.py   | 206 ++++++++++++++------
 src/calibre/gui2/dialogs/metadata_bulk.py   |   9 +-
 src/calibre/gui2/dialogs/metadata_single.py |  28 ++-
 src/calibre/library/custom_columns.py       |  21 +-
 4 files changed, 179 insertions(+), 85 deletions(-)

diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py
index 90ac7dbbaf..6bb481ddec 100644
--- a/src/calibre/gui2/custom_column_widgets.py
+++ b/src/calibre/gui2/custom_column_widgets.py
@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import re, sys
 from functools import partial
 
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
@@ -162,7 +162,6 @@ class DateTime(Base):
             val = qt_to_dt(val)
         return val
 
-
 class Comments(Base):
 
     def setup_ui(self, parent):
@@ -199,11 +198,7 @@ class Text(Base):
             w = EnComboBox(parent)
             w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
             w.setMinimumContentsLength(25)
-
-
-
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
-                w]
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
 
     def initialize(self, book_id):
         val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
@@ -222,7 +217,6 @@ class Text(Base):
             if idx is not None:
                 self.widgets[1].setCurrentIndex(idx)
 
-
     def setter(self, val):
         if self.col_metadata['is_multiple']:
             if not val:
@@ -241,6 +235,58 @@ class Text(Base):
             val = None
         return val
 
+class Series(Base):
+
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent))
+        w = QDoubleSpinBox(parent)
+        w.setRange(-100., float(sys.maxint))
+        w.setDecimals(2)
+        w.setSpecialValueText(_('Undefined'))
+        w.setSingleStep(1)
+        self.idx_widget=w
+        self.widgets.append(w)
+
+    def initialize(self, book_id):
+        val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
+        s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True)
+        if s_index is None:
+            s_index = 0.0
+        self.idx_widget.setValue(s_index)
+        self.initial_index = s_index
+        self.initial_val = val
+        val = self.normalize_db_val(val)
+        idx = None
+        for i, c in enumerate(self.all_values):
+            if c == val:
+                idx = i
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+        if idx is not None:
+            self.widgets[1].setCurrentIndex(idx)
+
+    def commit(self, book_id, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        s_index = self.idx_widget.value()
+        if val != self.initial_val or s_index != self.initial_index:
+            if s_index == 0.0:
+                if tweaks['series_index_auto_increment'] == 'next':
+                    s_index = self.db.get_next_cc_series_num_for(val,
+                                                             num=self.col_id)
+                else:
+                    s_index = None
+            self.db.set_custom(book_id, val, extra=s_index,
+                               num=self.col_id, notify=notify)
+
 widgets = {
         'bool' : Bool,
         'rating' : Rating,
@@ -249,6 +295,7 @@ widgets = {
         'datetime': DateTime,
         'text' : Text,
         'comments': Comments,
+        'series': Series,
 }
 
 def field_sort(y, z, x=None):
@@ -257,35 +304,63 @@ def field_sort(y, z, x=None):
     n2 = 'zzzzz' if m2['datatype'] == 'comments' else m2['name']
     return cmp(n1.lower(), n2.lower())
 
-def populate_single_metadata_page(left, right, db, book_id, parent=None):
+def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, parent=None):
+    def widget_factory(type, col):
+        if bulk:
+            w = bulk_widgets[type](db, col, parent)
+        else:
+            w = widgets[type](db, col, parent)
+        w.initialize(book_id)
+        return w
     x = db.custom_column_num_map
     cols = list(x)
     cols.sort(cmp=partial(field_sort, x=x))
+    count_non_comment = len([c for c in cols if x[c]['datatype'] != 'comments'])
+
+    layout.setColumnStretch(1, 10)
+    if two_column:
+        turnover_point = (count_non_comment+1)/2
+        layout.setColumnStretch(3, 10)
+    else:
+        # Avoid problems with multi-line widgets
+        turnover_point = count_non_comment + 1000
     ans = []
-    for i, col in enumerate(cols):
-        w = widgets[x[col]['datatype']](db, col, parent)
+    column = row = 0
+    for col in cols:
+        dt = x[col]['datatype']
+        if dt == 'comments':
+            continue
+        w = widget_factory(dt, col)
         ans.append(w)
-        w.initialize(book_id)
-        layout = left if i%2 == 0 else right
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            w.widgets[0].setBuddy(w.widgets[1])
-            for c, widget in enumerate(w.widgets):
-                layout.addWidget(widget, row, c)
+        for c in range(0, len(w.widgets), 2):
+            w.widgets[c].setBuddy(w.widgets[c+1])
+            layout.addWidget(w.widgets[c], row, column)
+            layout.addWidget(w.widgets[c+1], row, column+1)
+            row += 1
+        if row >= turnover_point:
+            column += 2
+            turnover_point = count_non_comment + 1000
+            row = 0
+    if not bulk: # Add the comments fields
+        column = 0
+        for col in cols:
+            dt = x[col]['datatype']
+            if dt != 'comments':
+                continue
+            w = widget_factory(dt, col)
+            ans.append(w)
+            layout.addWidget(w.widgets[0], row, column, 1, 2)
+            if two_column and column == 0:
+                column = 2
+                continue
+            column = 0
+            row += 1
     items = []
     if len(ans) > 0:
         items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
             QSizePolicy.Expanding))
-        left.addItem(items[-1], left.rowCount(), 0, 1, 1)
-        left.setRowStretch(left.rowCount()-1, 100)
-    if len(ans) > 1:
-         items.append(QSpacerItem(10, 100, QSizePolicy.Minimum,
-             QSizePolicy.Expanding))
-         right.addItem(items[-1], left.rowCount(), 0, 1, 1)
-         right.setRowStretch(right.rowCount()-1, 100)
-
+        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
+        layout.setRowStretch(layout.rowCount()-1, 100)
     return ans, items
 
 class BulkBase(Base):
@@ -342,6 +417,47 @@ class BulkRating(BulkBase, Rating):
 class BulkDateTime(BulkBase, DateTime):
     pass
 
+class BulkSeries(BulkBase):
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
+        self.idx_widget=QCheckBox(parent)
+        self.widgets.append(self.idx_widget)
+
+    def initialize(self, book_id):
+        self.idx_widget.setChecked(False)
+        for c in self.all_values:
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+
+    def commit(self, book_ids, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        update_indices = self.idx_widget.checkState()
+        if val != '':
+            for book_id in book_ids:
+                if update_indices:
+                    if tweaks['series_index_auto_increment'] == 'next':
+                        s_index = self.db.get_next_cc_series_num_for\
+                                                        (val, num=self.col_id)
+                    else:
+                        s_index = 1.0
+                else:
+                    s_index = self.db.get_custom_extra(book_id, num=self.col_id,
+                                                       index_is_id=True)
+                self.db.set_custom(book_id, val, extra=s_index,
+                                   num=self.col_id, notify=notify)
+
+    def process_each_book(self):
+        return True
+
 class RemoveTags(QWidget):
 
     def __init__(self, parent, values):
@@ -431,35 +547,5 @@ bulk_widgets = {
         'float': BulkFloat,
         'datetime': BulkDateTime,
         'text' : BulkText,
-}
-
-def populate_bulk_metadata_page(layout, db, book_ids, parent=None):
-    x = db.custom_column_num_map
-    cols = list(x)
-    cols.sort(cmp=partial(field_sort, x=x))
-    ans = []
-    for i, col in enumerate(cols):
-        dt = x[col]['datatype']
-        if dt == 'comments':
-            continue
-        w = bulk_widgets[dt](db, col, parent)
-        ans.append(w)
-        w.initialize(book_ids)
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            for c in range(0, len(w.widgets), 2):
-                w.widgets[c].setBuddy(w.widgets[c+1])
-                layout.addWidget(w.widgets[c], row, 0)
-                layout.addWidget(w.widgets[c+1], row, 1)
-                row += 1
-    items = []
-    if len(ans) > 0:
-        items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
-            QSizePolicy.Expanding))
-        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
-        layout.setRowStretch(layout.rowCount()-1, 100)
-
-    return ans, items
-
+        'series': BulkSeries,
+}
\ No newline at end of file
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 8b27ff1999..9fcfe13253 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -10,7 +10,7 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, \
     authors_to_string
-from calibre.gui2.custom_column_widgets import populate_bulk_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -44,15 +44,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.central_widget.tabBar().setVisible(False)
         else:
             self.create_custom_column_editors()
-
         self.exec_()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = QGridLayout()
-
-        self.custom_column_widgets, self.__cc_spacers = populate_bulk_metadata_page(
-                layout, self.db, self.ids, w)
+        self.custom_column_widgets, self.__cc_spacers = \
+            populate_metadata_page(layout, self.db, self.ids, parent=w,
+                                   two_column=False, bulk=True)
         w.setLayout(layout)
         self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets
diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py
index 96323ac596..84b601776e 100644
--- a/src/calibre/gui2/dialogs/metadata_single.py
+++ b/src/calibre/gui2/dialogs/metadata_single.py
@@ -32,7 +32,7 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import qt_to_dt
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.dialogs.config.social import SocialMetadata
-from calibre.gui2.custom_column_widgets import populate_single_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class CoverFetcher(QThread):
 
@@ -420,23 +420,19 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
-        top_layout = QHBoxLayout()
-        top_layout.setSpacing(20)
-        left_layout = QGridLayout()
-        right_layout = QGridLayout()
-        top_layout.addLayout(left_layout)
-
-        self.custom_column_widgets, self.__cc_spacers = populate_single_metadata_page(
-                left_layout, right_layout, self.db, self.id, w)
-        top_layout.addLayout(right_layout)
-        sip.delete(w.layout())
-        w.setLayout(top_layout)
-        self.__custom_col_layouts = [top_layout, left_layout, right_layout]
+        layout = w.layout()
+        self.custom_column_widgets, self.__cc_spacers = \
+                    populate_metadata_page(layout, self.db, self.id,
+                                           parent=w, bulk=False, two_column=True)
+        self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets
         for i in range(len(ans)-1):
-            w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[-1])
-
-
+            if len(ans[i+1].widgets) == 2:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1])
+            else:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0])
+            for c in range(2, len(ans[i].widgets), 2):
+                w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1])
 
     def validate_isbn(self, isbn):
         isbn = unicode(isbn).strip()
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 57b355045a..52084fcda1 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -173,6 +173,19 @@ class CustomColumns(object):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
 
+    def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        # add future datatypes with an extra column here
+        if data['datatype'] not in ['series']:
+            return None
+        ign,lt = self.custom_table_names(data['num'])
+        idx = idx if index_is_id else self.id(idx)
+        return self.conn.get('''SELECT extra FROM %s
+                                WHERE book=?'''%lt, (idx,), all=False)
+
     # convenience methods for tag editing
     def get_custom_items_with_ids(self, label=None, num=None):
         if label is not None:
@@ -237,7 +250,7 @@ class CustomColumns(object):
             return 1.0
         # get the label of the associated series number table
         series_num = self.conn.get('''
-                SELECT MAX({lt}.s_index) FROM {lt}
+                SELECT MAX({lt}.extra) FROM {lt}
                 WHERE {lt}.book IN (SELECT book FROM {lt} where value=?)
                 '''.format(lt=lt), (series_id,), all=False)
         if series_num is None:
@@ -343,7 +356,7 @@ class CustomColumns(object):
                                                         (id_, xid), all=False):
                     if data['datatype'] == 'series':
                         self.conn.execute(
-                            '''INSERT INTO %s(book, value, s_index)
+                            '''INSERT INTO %s(book, value, extra)
                                VALUES (?,?,?)'''%lt, (id_, xid, extra))
                         self.data.set(id_, self.FIELD_MAP[data['num']]+1,
                                       extra, row_is_id=True)
@@ -401,7 +414,7 @@ class CustomColumns(object):
                     custom_{num}
                 '''.format(query=query%table, lt=lt, table=table, num=data['num'])
                 if data['datatype'] == 'series':
-                    line += ''',(SELECT s_index FROM {lt} WHERE {lt}.book=books.id)
+                    line += ''',(SELECT extra FROM {lt} WHERE {lt}.book=books.id)
                         custom_index_{num}'''.format(lt=lt, num=data['num'])
             else:
                 line = '''
@@ -438,7 +451,7 @@ class CustomColumns(object):
         table, lt = self.custom_table_names(num)
         if normalized:
             if datatype == 'series':
-                s_index = 's_index REAL,'
+                s_index = 'extra REAL,'
             else:
                 s_index = ''
             lines = [

From ee58333dcb6dfa1306adb212a386f43d3cffb5d7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 20 Jun 2010 19:22:26 +0100
Subject: [PATCH 03/31] Fix samsung android driver

---
 src/calibre/devices/android/driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py
index 0bbdf0f22c..49c41b4e57 100644
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@@ -45,8 +45,8 @@ class ANDROID(USBMS):
             'GT-I5700', 'SAMSUNG']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD',
-            'PROD_GT-I9000']
-    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PROD_GT-I9000_CARD']
+            'PR OD_GT-I9000']
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PR OD_GT-I9000_CARD']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

From 2d5daf4a02e6c74387213f9dc00eebc9fdfbb171 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 20 Jun 2010 20:43:43 +0100
Subject: [PATCH 04/31] Enable completion for on-library editing of series
 columns

---
 src/calibre/gui2/library/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index b949306294..8245c2d188 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -347,7 +347,7 @@ class BooksView(QTableView): # {{{
                     self.setItemDelegateForColumn(cm.index(colhead), delegate)
                 elif cc['datatype'] == 'comments':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_comments_delegate)
-                elif cc['datatype'] == 'text':
+                elif cc['datatype'] in ('text', 'series'):
                     if cc['is_multiple']:
                         self.setItemDelegateForColumn(cm.index(colhead), self.tags_delegate)
                     else:

From 0dc63fa59f7531dc655b14b8872ff89f24f7d46f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 21 Jun 2010 12:17:48 +0100
Subject: [PATCH 05/31] Fix problems with coordination between tags browser and
 search window. Search from the tag browser, then type in the search box. Tag
 browser was not cleared.

---
 src/calibre/gui2/search_box.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py
index 35bf7374a0..ef0c6b1455 100644
--- a/src/calibre/gui2/search_box.py
+++ b/src/calibre/gui2/search_box.py
@@ -56,7 +56,8 @@ class SearchBox2(QComboBox):
     To use this class:
 
         * Call initialize()
-        * Connect to the search() and cleared() signals from this widget
+        * Connect to the search() and cleared() signals from this widget.
+        * Connect to the cleared() signal to know when the box content changes
         * Call search_done() after every search is complete
         * Use clear() to clear back to the help message
     '''
@@ -75,6 +76,7 @@ class SearchBox2(QComboBox):
                 type=Qt.DirectConnection)
         self.line_edit.mouse_released.connect(self.mouse_released,
                 type=Qt.DirectConnection)
+        self.activated.connect(self.history_selected)
         self.setEditable(True)
         self.help_state = False
         self.as_you_type = True
@@ -139,6 +141,9 @@ class SearchBox2(QComboBox):
 
     def key_pressed(self, event):
         self.normalize_state()
+        if self._in_a_search:
+            self.emit(SIGNAL('changed()'))
+            self._in_a_search = False
         if event.key() in (Qt.Key_Return, Qt.Key_Enter):
             self.do_search()
         self.timer = self.startTimer(self.__class__.INTERVAL)
@@ -154,6 +159,10 @@ class SearchBox2(QComboBox):
             self.timer = None
             self.do_search()
 
+    def history_selected(self, text):
+        self.emit(SIGNAL('changed()'))
+        self.do_search()
+
     @property
     def smart_text(self):
         text = unicode(self.currentText()).strip()
@@ -345,6 +354,7 @@ class SearchBoxMixin(object):
         self.search.initialize('main_search_history', colorize=True,
                 help_text=_('Search (For Advanced Search click the button to the left)'))
         self.connect(self.search, SIGNAL('cleared()'), self.search_box_cleared)
+        self.connect(self.search, SIGNAL('changed()'), self.search_box_changed)
         self.connect(self.clear_button, SIGNAL('clicked()'), self.search.clear)
         QObject.connect(self.advanced_search_button, SIGNAL('clicked(bool)'),
                         self.do_advanced_search)
@@ -364,6 +374,9 @@ class SearchBoxMixin(object):
         self.saved_search.clear_to_help()
         self.set_number_of_books_shown()
 
+    def search_box_changed(self):
+        self.tags_view.clear()
+
     def do_advanced_search(self, *args):
         d = SearchDialog(self)
         if d.exec_() == QDialog.Accepted:

From eaf9c93e9e7a47f917e4a7c5778626861e3e557b Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 22 Jun 2010 08:59:37 +0100
Subject: [PATCH 06/31] Add cancel button to ChooseFormatDialog, and make the
 send to device menus honor it.

---
 src/calibre/gui2/device.py                | 5 +++--
 src/calibre/gui2/dialogs/choose_format.ui | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 33191d1773..850396bc5d 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -10,7 +10,7 @@ from functools import partial
 from binascii import unhexlify
 
 from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \
-                     Qt, pyqtSignal, QColor, QPainter
+                     Qt, pyqtSignal, QColor, QPainter, QDialog
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre.customize.ui import available_input_formats, available_output_formats, \
@@ -814,7 +814,8 @@ class DeviceMixin(object): # {{{
         if specific:
             d = ChooseFormatDialog(self, _('Choose format to send to device'),
                                 self.device_manager.device.settings().format_map)
-            d.exec_()
+            if d.exec_() != QDialog.Accepted:
+                return
             if d.format():
                 fmt = d.format().lower()
         dest, sub_dest = dest.split(':')
diff --git a/src/calibre/gui2/dialogs/choose_format.ui b/src/calibre/gui2/dialogs/choose_format.ui
index 0ae0fa8b94..50dd4b4fc1 100644
--- a/src/calibre/gui2/dialogs/choose_format.ui
+++ b/src/calibre/gui2/dialogs/choose_format.ui
@@ -39,7 +39,7 @@
       <enum>Qt::Horizontal</enum>
      </property>
      <property name="standardButtons" >
-      <set>QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Ok|QDialogButtonBox::Cancel</set>
      </property>
     </widget>
    </item>

From 3228374bf935e1b971e197053257762fa5c6488f Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 22 Jun 2010 22:32:19 +0100
Subject: [PATCH 07/31] Fix yet another problem with sony collections. This one
 was provoked by doing a sync booklist just after loading the cache. The
 problem was that the collections had not been computed, so they were all
 removed. The bug could happen if only one book was out of sync.

---
 src/calibre/devices/prs505/sony_cache.py | 109 ++++++++++-------------
 src/calibre/gui2/device.py               |   5 ++
 src/calibre/gui2/library/models.py       |   2 +-
 3 files changed, 53 insertions(+), 63 deletions(-)

diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py
index e7d0e4686c..a704824a3f 100644
--- a/src/calibre/devices/prs505/sony_cache.py
+++ b/src/calibre/devices/prs505/sony_cache.py
@@ -9,7 +9,6 @@ import os, time
 from pprint import pprint
 from base64 import b64decode
 from uuid import uuid4
-
 from lxml import etree
 
 from calibre import prints, guess_type
@@ -151,15 +150,14 @@ class XMLCache(object):
                 else:
                     seen.add(title)
 
-    def get_playlist_map(self):
-        debug_print('Start get_playlist_map')
+    def build_playlist_id_map(self):
+        debug_print('Start build_playlist_id_map')
         ans = {}
         self.ensure_unique_playlist_titles()
         debug_print('after ensure_unique_playlist_titles')
         self.prune_empty_playlists()
-        debug_print('get_playlist_map loop')
         for i, root in self.record_roots.items():
-            debug_print('get_playlist_map loop', i)
+            debug_print('build_playlist_id_map loop', i)
             id_map = self.build_id_map(root)
             ans[i] = []
             for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -170,9 +168,23 @@ class XMLCache(object):
                     if record is not None:
                         items.append(record)
                 ans[i].append((playlist.get('title'), items))
-        debug_print('end get_playlist_map')
+        debug_print('end build_playlist_id_map')
         return ans
 
+    def build_id_playlist_map(self, bl_index):
+        debug_print('Start build_id_playlist_map')
+        pmap = self.build_playlist_id_map()[bl_index]
+        playlist_map = {}
+        for title, records in pmap:
+            for record in records:
+                path = record.get('path', None)
+                if path:
+                    if path not in playlist_map:
+                        playlist_map[path] = set()
+                    playlist_map[path].add(title)
+        debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
+        return playlist_map
+
     def get_or_create_playlist(self, bl_idx, title):
         root = self.record_roots[bl_idx]
         for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -192,8 +204,7 @@ class XMLCache(object):
     # }}}
 
     def fix_ids(self): # {{{
-        if DEBUG:
-            debug_print('Running fix_ids()')
+        debug_print('Running fix_ids()')
 
         def ensure_numeric_ids(root):
             idmap = {}
@@ -276,38 +287,19 @@ class XMLCache(object):
     def update_booklist(self, bl, bl_index):
         if bl_index not in self.record_roots:
             return
-        if DEBUG:
-            debug_print('Updating JSON cache:', bl_index)
+        debug_print('Updating JSON cache:', bl_index)
+        playlist_map = self.build_id_playlist_map(bl_index)
         root = self.record_roots[bl_index]
-        pmap = self.get_playlist_map()[bl_index]
-        playlist_map = {}
-        for title, records in pmap:
-            for record in records:
-                path = record.get('path', None)
-                if path:
-                    if path not in playlist_map:
-                        playlist_map[path] = []
-                    playlist_map[path].append(title)
-
         lpath_map = self.build_lpath_map(root)
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
                 title = record.get('title', None)
                 if title is not None and title != book.title:
-                    if DEBUG:
-                        debug_print('Renaming title', book.title, 'to', title)
+                    debug_print('Renaming title', book.title, 'to', title)
                     book.title = title
-# We shouldn't do this for Sonys, because the reader strips
-# all but the first author.
-#                authors = record.get('author', None)
-#                if authors is not None:
-#                    authors = string_to_authors(authors)
-#                    if authors != book.authors:
-#                        if DEBUG:
-#                            prints('Renaming authors', book.authors, 'to',
-#                                    authors)
-#                        book.authors = authors
+                    # Don't set the author, because the reader strips all but
+                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -318,45 +310,45 @@ class XMLCache(object):
                             book.thumbnail = raw
                             break
                     break
-                if book.lpath in playlist_map:
-                    tags = playlist_map[book.lpath]
-                    book.device_collections = tags
+                book.device_collections = list(playlist_map.get(book.lpath, set()))
         debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
-        debug_print('Starting update XML from JSON')
-        playlist_map = self.get_playlist_map()
-
+        debug_print('In update. Starting update XML from JSON')
         for i, booklist in booklists.items():
-            if DEBUG:
-                debug_print('Updating XML Cache:', i)
+            playlist_map = self.build_id_playlist_map(i)
+            debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
             lpath_map = self.build_lpath_map(root)
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
-#                record = self.book_by_lpath(book.lpath, root)
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
                 self.update_text_record(record, book, path, i)
-
-            bl_pmap = playlist_map[i]
-            self.update_playlists(i, root, booklist, bl_pmap,
-                    collections_attributes)
-
-        self.fix_ids()
-
-        # This is needed to update device_collections
+                # Ensure the collections in the XML database are recorded for
+                # this book
+                if book.device_collections is None:
+                    book.device_collections = []
+                book.device_collections = list(set(book.device_collections) |
+                                            playlist_map.get(book.lpath, set()))
+            self.update_playlists(i, root, booklist, collections_attributes)
+        # Update the device collections because update playlist could have added
+        # some new ones.
+        debug_print('In update/ Starting refresh of device_collections')
         for i, booklist in booklists.items():
-            self.update_booklist(booklist, i)
+            playlist_map = self.build_id_playlist_map(i)
+            for book in booklist:
+                book.device_collections = list(set(book.device_collections) |
+                                            playlist_map.get(book.lpath, set()))
+        self.fix_ids()
         debug_print('Finished update XML from JSON')
 
-    def update_playlists(self, bl_index, root, booklist, playlist_map,
-            collections_attributes):
-        debug_print('Starting update_playlists')
+    def update_playlists(self, bl_index, root, booklist, collections_attributes):
+        debug_print('Starting update_playlists', collections_attributes)
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
@@ -372,10 +364,8 @@ class XMLCache(object):
                     rec.set('id', str(self.max_id(root)+1))
             ids = [x.get('id', None) for x in records]
             if None in ids:
-                if DEBUG:
-                    debug_print('WARNING: Some <text> elements do not have ids')
-                    ids = [x for x in ids if x is not None]
-
+                debug_print('WARNING: Some <text> elements do not have ids')
+                ids = [x for x in ids if x is not None]
             playlist = self.get_or_create_playlist(bl_index, category)
             playlist_ids = []
             for item in playlist:
@@ -544,10 +534,5 @@ class XMLCache(object):
                         break
                 self.namespaces[i] = ns
 
-#        if DEBUG:
-#            debug_print('Found nsmaps:')
-#            pprint(self.nsmaps)
-#            debug_print('Found namespaces:')
-#            pprint(self.namespaces)
     # }}}
 
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 850396bc5d..836105aba9 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1228,6 +1228,11 @@ class DeviceMixin(object): # {{{
             return
         cp, fs = job.result
         self.location_view.model().update_devices(cp, fs)
+        # reset the views so that up-to-date info is shown. These need to be
+        # here because the sony driver updates collections in sync_booklists
+        self.memory_view.reset()
+        self.card_a_view.reset()
+        self.card_b_view.reset()
 
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 435b5c4c07..a85462e22d 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -1118,7 +1118,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
-                    return QVariant(', '.join(tags))
+                    return QVariant(', '.join(sorted(tags, key=str.lower)))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
                 return QVariant(_('Marked for deletion'))

From e79f9e06ae121975bcf4130a40737c6d37256cbf Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 02:02:39 -0400
Subject: [PATCH 08/31] Fix resue of 'occasion' causing postprocess file
 plugins from using the new file.

---
 src/calibre/customize/ui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py
index 8397827fbb..14d22d5017 100644
--- a/src/calibre/customize/ui.py
+++ b/src/calibre/customize/ui.py
@@ -151,13 +151,13 @@ def reread_filetype_plugins():
 
 
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
-    occasion = {'import':_on_import, 'preprocess':_on_preprocess,
+    occasion_plugins = {'import':_on_import, 'preprocess':_on_preprocess,
                 'postprocess':_on_postprocess}[occasion]
     customization = config['plugin_customization']
     if ft is None:
         ft = os.path.splitext(path_to_file)[-1].lower().replace('.', '')
     nfp = path_to_file
-    for plugin in occasion.get(ft, []):
+    for plugin in occasion_plugins.get(ft, []):
         if is_disabled(plugin):
             continue
         plugin.site_customization = customization.get(plugin.name, '')

From 456eda2519da966e7160d738d0a2aad98011d964 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 23 Jun 2010 11:19:15 +0100
Subject: [PATCH 09/31] Kovid-suggested changes to series columns

---
 src/calibre/gui2/library/models.py    | 13 ++----------
 src/calibre/library/caches.py         |  8 +++----
 src/calibre/library/cli.py            | 30 +++++++++++++++++++++++----
 src/calibre/library/custom_columns.py |  4 ++--
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index a85462e22d..008f024aae 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -21,6 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
@@ -708,17 +709,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     return False
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
-            val = unicode(value.toString()).strip()
-            pat = re.compile(r'\[([.0-9]+)\]')
-            match = pat.search(val)
-            if match is not None:
-                val = pat.sub('', val).strip()
-                s_index = float(match.group(1))
-            elif val:
-                if tweaks['series_index_auto_increment'] == 'next':
-                    s_index = self.db.get_next_cc_series_num_for(val, label=label)
-                else:
-                    s_index = 1.0
+            val, s_index = parse_series_string(self.db, label, value.toString())
         self.db.set_custom(self.db.id(row), val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
         return True
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index d4afaabcdc..06cf07bb67 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -620,6 +620,10 @@ class ResultCache(SearchQueryParser):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
+
+        if self.first_sort:
+            subsort = True
+            self.first_sort = False
         if self.field_metadata[field]['is_custom']:
             if self.field_metadata[field]['datatype'] == 'series':
                 fcmp = functools.partial(self.seriescmp,
@@ -638,10 +642,6 @@ class ResultCache(SearchQueryParser):
         else:
             fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
                                      subsort=subsort, asstr=as_string)
-
-        if self.first_sort:
-            subsort = True
-            self.first_sort = False
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]
 
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 3f71c98238..058b879b55 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO
+import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
 from calibre import terminal_controller, preferred_encoding, prints
-from calibre.utils.config import OptionParser, prefs
+from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
@@ -680,9 +680,31 @@ def command_catalog(args, dbpath):
 
 # end of GR additions
 
+def parse_series_string(db, label, value):
+    val = unicode(value).strip()
+    s_index = None
+    pat = re.compile(r'\[([.0-9]+)\]')
+    match = pat.search(val)
+    if match is not None:
+        val = pat.sub('', val).strip()
+        s_index = float(match.group(1))
+    elif val:
+        if tweaks['series_index_auto_increment'] == 'next':
+            s_index = db.get_next_cc_series_num_for(val, label=label)
+        else:
+            s_index = 1.0
+    return val, s_index
+
 def do_set_custom(db, col, id_, val, append):
-    db.set_custom(id_, val, label=col, append=append)
-    prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
+    if db.custom_column_label_map[col]['datatype'] == 'series':
+        val, s_index = parse_series_string(db, col, val)
+        db.set_custom(id_, val, extra=s_index, label=col, append=append)
+        prints('Data set to: %r[%4.2f]'%
+               (db.get_custom(id_, label=col, index_is_id=True),
+                db.get_custom_extra(id_, label=col, index_is_id=True)))
+    else:
+        db.set_custom(id_, val, label=col, append=append)
+        prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
 
 def set_custom_option_parser():
     parser = get_parser(_(
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 52084fcda1..e039f5a817 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -308,8 +308,8 @@ class CustomColumns(object):
             self.conn.commit()
         return changed
 
-    def set_custom(self, id_, val, extra=None, label=None, num=None,
-                   append=False, notify=True):
+    def set_custom(self, id_, val, label=None, num=None,
+                   append=False, notify=True, extra=None):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:

From 0c3d4511f831d4cfd05584c148b2fb4c259b5221 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 23 Jun 2010 12:05:22 +0100
Subject: [PATCH 10/31] Fix problem where collections are not built if the user
 spells the db fields using uppercase letters.

---
 src/calibre/devices/prs505/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index 5860826778..023416bdf2 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -145,7 +145,7 @@ class PRS505(USBMS):
                 blists[i] = booklists[i]
         opts = self.settings()
         if opts.extra_customization:
-            collections = [x.strip() for x in
+            collections = [x.lower().strip() for x in
                     opts.extra_customization.split(',')]
         else:
             collections = []

From d344bf0d3d13e72b0ffce9383bc44d15df8a0f6e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 07:02:20 -0600
Subject: [PATCH 11/31] Cover browser: Set aspect ratio of covers to 3:4
 instead of 2:3. Crop rather than distort covers whoose aspect ratio is
 different from this. Antialias the rendering of text in the central cover

---
 src/calibre/gui2/pictureflow/pictureflow.cpp | 88 ++++++--------------
 src/calibre/gui2/pictureflow/pictureflow.h   | 11 ---
 src/calibre/gui2/pictureflow/pictureflow.sip |  4 -
 3 files changed, 27 insertions(+), 76 deletions(-)

diff --git a/src/calibre/gui2/pictureflow/pictureflow.cpp b/src/calibre/gui2/pictureflow/pictureflow.cpp
index d1434e763c..a100f60e75 100644
--- a/src/calibre/gui2/pictureflow/pictureflow.cpp
+++ b/src/calibre/gui2/pictureflow/pictureflow.cpp
@@ -75,10 +75,6 @@
 
 #include <QDebug>
 
-// uncomment this to enable bilinear filtering for texture mapping
-// gives much better rendering, at the cost of memory space
-// #define PICTUREFLOW_BILINEAR_FILTER
-
 // for fixed-point arithmetic, we need minimum 32-bit long
 // long long (64-bit) might be useful for multiplication and division
 typedef long PFreal;
@@ -376,7 +372,6 @@ private:
   int slideWidth;
   int slideHeight;
   int fontSize;
-  int zoom;
   int queueLength;
 
   int centerIndex;
@@ -401,6 +396,7 @@ private:
 
   void recalc(int w, int h);
   QRect renderSlide(const SlideInfo &slide, int alpha=256, int col1=-1, int col=-1);
+  QRect renderCenterSlide(const SlideInfo &slide);
   QImage* surface(int slideIndex);
   void triggerRender();
   void resetSlides();
@@ -414,7 +410,6 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_)
   slideWidth = 200;
   slideHeight = 200;
   fontSize = 10;
-  zoom = 100;
 
   centerIndex = 0;
   queueLength = queueLength_;
@@ -464,21 +459,6 @@ void PictureFlowPrivate::setSlideSize(QSize size)
   triggerRender();
 }
 
-int PictureFlowPrivate::zoomFactor() const
-{
-  return zoom;
-}
-
-void PictureFlowPrivate::setZoomFactor(int z)
-{
-  if(z <= 0)
-    return;
-
-  zoom = z;
-  recalc(buffer.width(), buffer.height());
-  triggerRender();
-}
-
 QImage PictureFlowPrivate::slide(int index) const
 {
   return slideImages->image(index);
@@ -554,7 +534,8 @@ void PictureFlowPrivate::resize(int w, int h)
   if (w < 10) w = 10;
   if (h < 10) h = 10;
   slideHeight = int(float(h)/REFLECTION_FACTOR);
-  slideWidth = int(float(slideHeight) * 2/3.);
+  slideWidth = int(float(slideHeight) * 3./4.);
+  //qDebug() << slideHeight << "x" << slideWidth;
   fontSize = MAX(int(h/15.), 12);
   recalc(w, h);
   resetSlides();
@@ -595,15 +576,12 @@ void PictureFlowPrivate::resetSlides()
   }
 }
 
-#define BILINEAR_STRETCH_HOR 4
-#define BILINEAR_STRETCH_VER 4
-
 static QImage prepareSurface(QImage img, int w, int h)
 {
   Qt::TransformationMode mode = Qt::SmoothTransformation;
-  img = img.scaled(w, h, Qt::IgnoreAspectRatio, mode);
+  img = img.scaled(w, h, Qt::KeepAspectRatioByExpanding, mode);
 
-  // slightly larger, to accomodate for the reflection
+  // slightly larger, to accommodate for the reflection
   int hs = int(h * REFLECTION_FACTOR);
   int hofs = 0;
 
@@ -633,12 +611,6 @@ static QImage prepareSurface(QImage img, int w, int h)
       result.setPixel(h+hofs+y, x, qRgb(r, g, b));
     }
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int hh = BILINEAR_STRETCH_VER*hs;
-  int ww = BILINEAR_STRETCH_HOR*w;
-  result = result.scaled(hh, ww, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
-#endif
-
   return result;
 }
 
@@ -699,8 +671,12 @@ void PictureFlowPrivate::render()
 
   int nleft = leftSlides.count();
   int nright = rightSlides.count();
+  QRect r;
 
-  QRect r = renderSlide(centerSlide);
+  if (step == 0) 
+      r = renderCenterSlide(centerSlide);
+  else
+      r = renderSlide(centerSlide);
   int c1 = r.left();
   int c2 = r.right();
 
@@ -813,7 +789,23 @@ static inline uint BYTE_MUL_RGB16_32(uint x, uint a) {
     return t;
 }
 
+QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) {
+  QImage* src = surface(slide.slideIndex);
+  if(!src)
+    return QRect();
 
+  int sw = src->height();
+  int sh = src->width();
+  int h = buffer.height();
+  QRect rect(buffer.width()/2 - sw/2, 0, sw, h-1);
+  int left = rect.left();
+
+  for(int x = 0; x < sh-1; x++)
+    for(int y = 0; y < sw; y++)
+      buffer.setPixel(left + y, 1+x, src->pixel(x, y));
+
+  return rect;
+}
 // Renders a slide to offscreen buffer. Returns a rect of the rendered area.
 // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent
 // col1 and col2 limit the column for rendering.
@@ -826,13 +818,8 @@ int col1, int col2)
 
   QRect rect(0, 0, 0, 0);  
   
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int sw = src->height() / BILINEAR_STRETCH_HOR;
-  int sh = src->width() / BILINEAR_STRETCH_VER;
-#else
   int sw = src->height();
   int sh = src->width();
-#endif
   int h = buffer.height();
   int w = buffer.width();
 
@@ -848,7 +835,7 @@ int col1, int col2)
   col1 = qMin(col1, w-1);
   col2 = qMin(col2, w-1);
 
-  int distance = h * 100 / zoom;
+  int distance = h;
   PFreal sdx = fcos(slide.angle);
   PFreal sdy = fsin(slide.angle);
   PFreal xs = slide.cx - slideWidth * sdx/2;
@@ -878,15 +865,9 @@ int col1, int col2)
     PFreal hitx = fmul(dist, rays[x]);
     PFreal hitdist = fdiv(hitx - slide.cx, sdx);
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int column = sw*BILINEAR_STRETCH_HOR/2 + (hitdist*BILINEAR_STRETCH_HOR >> PFREAL_SHIFT);
-    if(column >= sw*BILINEAR_STRETCH_HOR)
-      break;
-#else
     int column = sw/2 + (hitdist >> PFREAL_SHIFT);
     if(column >= sw)
       break;
-#endif
     if(column < 0)
       continue;
 
@@ -901,13 +882,8 @@ int col1, int col2)
     QRgb565* pixel2 = (QRgb565*)(buffer.scanLine(y2)) + x;
     int pixelstep = pixel2 - pixel1;
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int center = (sh*BILINEAR_STRETCH_VER/2);
-    int dy = dist*BILINEAR_STRETCH_VER / h;
-#else
     int center = sh/2;
     int dy = dist / h;
-#endif
     int p1 = center*PFREAL_ONE - dy/2;
     int p2 = center*PFREAL_ONE + dy/2;
 
@@ -1155,16 +1131,6 @@ void PictureFlow::setSlideSize(QSize size)
   d->setSlideSize(size);
 }
 
-int PictureFlow::zoomFactor() const
-{
-  return d->zoomFactor();
-}
-
-void PictureFlow::setZoomFactor(int z)
-{
-  d->setZoomFactor(z);
-}
-
 QImage PictureFlow::slide(int index) const
 {
   return d->slide(index);
diff --git a/src/calibre/gui2/pictureflow/pictureflow.h b/src/calibre/gui2/pictureflow/pictureflow.h
index 8cce025180..13477a8771 100644
--- a/src/calibre/gui2/pictureflow/pictureflow.h
+++ b/src/calibre/gui2/pictureflow/pictureflow.h
@@ -91,7 +91,6 @@ Q_OBJECT
 
   Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide)
   Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize)
-  Q_PROPERTY(int zoomFactor READ zoomFactor WRITE setZoomFactor)
 
 public:
   /*!
@@ -120,16 +119,6 @@ public:
   */  
   void setSlideSize(QSize size);
 
-  /*!
-    Sets the zoom factor (in percent).
-  */ 
-  void setZoomFactor(int zoom);
-
-  /*!
-    Returns the zoom factor (in percent).
-  */
-  int zoomFactor() const;
-
   /*!
     Clears any caches held to free up memory
   */
diff --git a/src/calibre/gui2/pictureflow/pictureflow.sip b/src/calibre/gui2/pictureflow/pictureflow.sip
index 9202dd8ad5..f7ba12cee7 100644
--- a/src/calibre/gui2/pictureflow/pictureflow.sip
+++ b/src/calibre/gui2/pictureflow/pictureflow.sip
@@ -40,10 +40,6 @@ public :
 
   void setSlideSize(QSize size);
 
-  void setZoomFactor(int zoom);
-
-  int zoomFactor() const;
-
   void clearCaches();
 
   virtual QImage slide(int index) const;

From 44be78af3d326e88ad22053191cebddff09a534e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 08:23:34 -0600
Subject: [PATCH 12/31] Allow calibredb ot set series index for series type
 custom columns

---
 src/calibre/gui2/library/models.py    | 13 ++----------
 src/calibre/library/caches.py         |  8 +++----
 src/calibre/library/cli.py            | 30 +++++++++++++++++++++++----
 src/calibre/library/custom_columns.py |  4 ++--
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 435b5c4c07..debdaa4151 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -21,6 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
@@ -708,17 +709,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     return False
                 val = qt_to_dt(val, as_utc=False)
         elif typ == 'series':
-            val = unicode(value.toString()).strip()
-            pat = re.compile(r'\[([.0-9]+)\]')
-            match = pat.search(val)
-            if match is not None:
-                val = pat.sub('', val).strip()
-                s_index = float(match.group(1))
-            elif val:
-                if tweaks['series_index_auto_increment'] == 'next':
-                    s_index = self.db.get_next_cc_series_num_for(val, label=label)
-                else:
-                    s_index = 1.0
+            val, s_index = parse_series_string(self.db, label, value.toString())
         self.db.set_custom(self.db.id(row), val, extra=s_index,
                            label=label, num=None, append=False, notify=True)
         return True
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index d4afaabcdc..06cf07bb67 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -620,6 +620,10 @@ class ResultCache(SearchQueryParser):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
+
+        if self.first_sort:
+            subsort = True
+            self.first_sort = False
         if self.field_metadata[field]['is_custom']:
             if self.field_metadata[field]['datatype'] == 'series':
                 fcmp = functools.partial(self.seriescmp,
@@ -638,10 +642,6 @@ class ResultCache(SearchQueryParser):
         else:
             fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
                                      subsort=subsort, asstr=as_string)
-
-        if self.first_sort:
-            subsort = True
-            self.first_sort = False
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]
 
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 3f71c98238..058b879b55 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO
+import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
 from calibre import terminal_controller, preferred_encoding, prints
-from calibre.utils.config import OptionParser, prefs
+from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
@@ -680,9 +680,31 @@ def command_catalog(args, dbpath):
 
 # end of GR additions
 
+def parse_series_string(db, label, value):
+    val = unicode(value).strip()
+    s_index = None
+    pat = re.compile(r'\[([.0-9]+)\]')
+    match = pat.search(val)
+    if match is not None:
+        val = pat.sub('', val).strip()
+        s_index = float(match.group(1))
+    elif val:
+        if tweaks['series_index_auto_increment'] == 'next':
+            s_index = db.get_next_cc_series_num_for(val, label=label)
+        else:
+            s_index = 1.0
+    return val, s_index
+
 def do_set_custom(db, col, id_, val, append):
-    db.set_custom(id_, val, label=col, append=append)
-    prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
+    if db.custom_column_label_map[col]['datatype'] == 'series':
+        val, s_index = parse_series_string(db, col, val)
+        db.set_custom(id_, val, extra=s_index, label=col, append=append)
+        prints('Data set to: %r[%4.2f]'%
+               (db.get_custom(id_, label=col, index_is_id=True),
+                db.get_custom_extra(id_, label=col, index_is_id=True)))
+    else:
+        db.set_custom(id_, val, label=col, append=append)
+        prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
 
 def set_custom_option_parser():
     parser = get_parser(_(
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 52084fcda1..e039f5a817 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -308,8 +308,8 @@ class CustomColumns(object):
             self.conn.commit()
         return changed
 
-    def set_custom(self, id_, val, extra=None, label=None, num=None,
-                   append=False, notify=True):
+    def set_custom(self, id_, val, label=None, num=None,
+                   append=False, notify=True, extra=None):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:

From d44154243e6e88082b70def653771c95459c8da4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 08:26:26 -0600
Subject: [PATCH 13/31] SONY driver: Handle mistaken upper casing of filed
 names when user specifies what fields to build colelctions from

---
 src/calibre/devices/prs505/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index 5860826778..023416bdf2 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -145,7 +145,7 @@ class PRS505(USBMS):
                 blists[i] = booklists[i]
         opts = self.settings()
         if opts.extra_customization:
-            collections = [x.strip() for x in
+            collections = [x.lower().strip() for x in
                     opts.extra_customization.split(',')]
         else:
             collections = []

From d0e85129ea6bab211d169895f8d902fab46a27f6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 08:40:03 -0600
Subject: [PATCH 14/31] Add entry ot FAQ about how collections on the SONY
 reader are managed

---
 src/calibre/manual/faq.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index a7968fc4e1..974cbf27fc 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -101,6 +101,17 @@ We just need some information from you:
 Once you send us the output for a particular operating system, support for the device in that operating system
 will appear in the next release of |app|.
 
+How does |app| manage collections on my SONY reader?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When you send a book to the device, |app| will create collections based on the metadata for that book. By default, collections are created
+from tags and series. You can control what metadata is used by going to Preferences->Plugins->Device Interface plugins and customizing
+the SONY device interface plugin. 
+
+You can edit collections on the device in the device view in |app| by double clicking or right clicking on the collections field.
+
+|app| will not delete already existing collections on your device. To ensure that the collections are based only on current |app| metadata,
+delete and resend the books to the device.
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 62210eb8ff3a38cd7f1f2037a2b74bcb24aaa7bf Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 23 Jun 2010 15:44:59 +0100
Subject: [PATCH 15/31] Prepare for device_collection editor

---
 src/calibre/gui2/dialogs/tag_list_editor.py |  2 +-
 src/calibre/gui2/tag_view.py                | 23 ++++++++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 1ec80f4b4a..73e81e2d99 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -11,7 +11,7 @@ from calibre.ebooks.metadata import title_sort
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, db, tag_to_match, category):
+    def __init__(self, window, db, tag_to_match, category, data, compare):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index daea4e86ea..6b4e8dbd88 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -15,6 +15,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
                      QPushButton, QWidget, QItemDelegate
 
+from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
 from calibre.library.field_metadata import TagsIcons
@@ -680,7 +681,27 @@ class TagBrowserMixin(object): # {{{
             self.tags_view.recount()
 
     def do_tags_list_edit(self, tag, category):
-        d = TagListEditor(self, self.library_view.model().db, tag, category)
+        db=self.library_view.model().db
+        if category == 'tags':
+            result = db.get_tags_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        elif category == 'series':
+            result = db.get_series_with_ids()
+            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
+        elif category == 'publisher':
+            result = db.get_publishers_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        else: # should be a custom field
+            self.cc_label = None
+            if category in db.field_metadata:
+                self.cc_label = db.field_metadata[category]['label']
+                result = self.db.get_custom_items_with_ids(label=self.cc_label)
+            else:
+                result = []
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+
+        d = TagListEditor(self, db=db, tag_to_match=tag, category=category,
+                          data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
             # Clean up everything, as information could have changed for many books.

From 8bb493275e2bcd15b0770669eae5c7b1964e65e1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 11:28:22 -0600
Subject: [PATCH 16/31] Fix #5953 (New recipe for london review of books)

---
 resources/images/news/lrb.png       | Bin 0 -> 315 bytes
 resources/images/news/lrb_payed.png | Bin 0 -> 315 bytes
 resources/recipes/lrb.recipe        |  40 ++++++++-------
 resources/recipes/lrb_payed.recipe  |  75 ++++++++++++++++++++++++++++
 4 files changed, 98 insertions(+), 17 deletions(-)
 create mode 100644 resources/images/news/lrb.png
 create mode 100644 resources/images/news/lrb_payed.png
 create mode 100644 resources/recipes/lrb_payed.recipe

diff --git a/resources/images/news/lrb.png b/resources/images/news/lrb.png
new file mode 100644
index 0000000000000000000000000000000000000000..da966d6a1ac856337112794325ee8ae9d1ed3e1c
GIT binary patch
literal 315
zcmeAS@N?(olHy`uVBq!ia0vp^0wB!60wlNoGJgf6n3BBRT^Rni_n+Ah<Z%{wL>2>S
z4={E+nQaFWEGuwK2hw1@3^B*n9tLtUJY5_^G|q3GcvAF$0tf4H8|$W7OVp|_%{rsl
zrjllT^5tee8>g(ATTGj|l~UGNOET<U!MdPo?;_istEHXxBnE%%jkQv;Yj<pH>g8b7
zGE(Y#&*OH`Z?jX@anVb=wI6k}{ru@V<>mee#s_Rt>RXRZl?U3XTH+c}l9E`GYL#4+
z3Zxi}3=9o)4a{_nj6)0!t&9w<3`~GrD+7a9(oE(k8glbfGSeziG#FVKnOGT`K{U*s
Sz_Sggfx*+&&t;ucLK6U2mS5KZ

literal 0
HcmV?d00001

diff --git a/resources/images/news/lrb_payed.png b/resources/images/news/lrb_payed.png
new file mode 100644
index 0000000000000000000000000000000000000000..da966d6a1ac856337112794325ee8ae9d1ed3e1c
GIT binary patch
literal 315
zcmeAS@N?(olHy`uVBq!ia0vp^0wB!60wlNoGJgf6n3BBRT^Rni_n+Ah<Z%{wL>2>S
z4={E+nQaFWEGuwK2hw1@3^B*n9tLtUJY5_^G|q3GcvAF$0tf4H8|$W7OVp|_%{rsl
zrjllT^5tee8>g(ATTGj|l~UGNOET<U!MdPo?;_istEHXxBnE%%jkQv;Yj<pH>g8b7
zGE(Y#&*OH`Z?jX@anVb=wI6k}{ru@V<>mee#s_Rt>RXRZl?U3XTH+c}l9E`GYL#4+
z3Zxi}3=9o)4a{_nj6)0!t&9w<3`~GrD+7a9(oE(k8glbfGSeziG#FVKnOGT`K{U*s
Sz_Sggfx*+&&t;ucLK6U2mS5KZ

literal 0
HcmV?d00001

diff --git a/resources/recipes/lrb.recipe b/resources/recipes/lrb.recipe
index 0076b3e697..4a203c80ae 100644
--- a/resources/recipes/lrb.recipe
+++ b/resources/recipes/lrb.recipe
@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 lrb.co.uk
 '''
@@ -8,32 +8,38 @@ lrb.co.uk
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class LondonReviewOfBooks(BasicNewsRecipe):
-    title                 = u'London Review of Books'
-    __author__            = u'Darko Miletic'
-    description           = u'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
-    category              = 'news, literature, England'
-    publisher             = 'London Review of Books'
-    oldest_article        = 7
+    title                 = 'London Review of Books (free)'
+    __author__            = 'Darko Miletic'
+    description           = 'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB ltd.'
+    oldest_article        = 15
     max_articles_per_feed = 100
     language              = 'en_GB'
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
+    publication_type      = 'magazine'
+    masthead_url          = 'http://www.lrb.co.uk/assets/images/lrb_logo_big.gif'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
 
-    conversion_options = {  
+    conversion_options = {
                              'comments'  : description
                             ,'tags'      : category
                             ,'language'  : language
                             ,'publisher' : publisher
                          }
-    
-    keep_only_tags = [dict(name='div' , attrs={'id'   :'main'})]
-    remove_tags = [
-                    dict(name='div' , attrs={'class':['pagetools','issue-nav-controls','nocss']})
-                   ,dict(name='div' , attrs={'id'   :['mainmenu','precontent','otherarticles']     })
-                   ,dict(name='span', attrs={'class':['inlineright','article-icons']})
-                   ,dict(name='ul'  , attrs={'class':'article-controls'})
-                   ,dict(name='p'   , attrs={'class':'meta-info'       })
-                  ]
+
+    keep_only_tags = [dict(attrs={'class':['article-body indent','letters','article-list']})]
+    remove_attributes = ['width','height']
 
     feeds = [(u'London Review of Books', u'http://www.lrb.co.uk/lrbrss.xml')]
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup('http://www.lrb.co.uk/')
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        if cover_item:
+           cover_url = 'http://www.lrb.co.uk' + cover_item.a.img['src']
+        return cover_url
+
diff --git a/resources/recipes/lrb_payed.recipe b/resources/recipes/lrb_payed.recipe
new file mode 100644
index 0000000000..4888f61cb6
--- /dev/null
+++ b/resources/recipes/lrb_payed.recipe
@@ -0,0 +1,75 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+lrb.co.uk
+'''
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonReviewOfBooksPayed(BasicNewsRecipe):
+    title                 = 'London Review of Books'
+    __author__            = 'Darko Miletic'
+    description           = 'Subscription content. Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB Ltd.'
+    max_articles_per_feed = 100
+    language              = 'en_GB'
+    no_stylesheets        = True
+    delay                 = 1
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    INDEX                 = 'http://www.lrb.co.uk'
+    LOGIN                 = INDEX + '/login'
+    masthead_url          = INDEX + '/assets/images/lrb_logo_big.gif'
+    needs_subscription    = True
+    publication_type      = 'magazine'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.LOGIN)
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        lrbtitle = self.title
+        if  cover_item:
+            self.cover_url = self.INDEX + cover_item.a.img['src']
+            content = self.INDEX + cover_item.a['href']
+            soup2 = self.index_to_soup(content)
+            sitem = soup2.find(attrs={'class':'article-list'})
+            lrbtitle = soup2.head.title.string
+            for item in sitem.findAll('a',attrs={'class':'title'}):
+                description = u''
+                title_prefix = u''
+                feed_link = item
+                if feed_link.has_key('href'):
+                    url   = self.INDEX + feed_link['href']
+                    title = title_prefix + self.tag_to_string(feed_link)
+                    date  = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+        return [(lrbtitle, articles)]
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    keep_only_tags = [dict(name='div' , attrs={'class':['article-body indent','letters']})]
+    remove_attributes = ['width','height']

From 344141ff45ad58fb1f71274ac77ed796326fcb4e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 11:43:49 -0600
Subject: [PATCH 17/31] E-book viewer: Handle font-face rules specify multiple
 families to be substituted

---
 src/calibre/ebooks/oeb/iterator.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/oeb/iterator.py b/src/calibre/ebooks/oeb/iterator.py
index 3fdd6aaf99..7f56cb4d2d 100644
--- a/src/calibre/ebooks/oeb/iterator.py
+++ b/src/calibre/ebooks/oeb/iterator.py
@@ -139,11 +139,18 @@ class EbookIterator(object):
                     if id != -1:
                         families = [unicode(f) for f in QFontDatabase.applicationFontFamilies(id)]
                         if family:
-                            family = family.group(1).strip().replace('"', '')
-                            bad_map[family] = families[0]
-                            if family not in families:
+                            family = family.group(1)
+                            specified_families = [x.strip().replace('"',
+                                '').replace("'", '') for x in family.split(',')]
+                            aliasing_ok = False
+                            for f in specified_families:
+                                bad_map[f] = families[0]
+                                if not aliasing_ok and f in families:
+                                    aliasing_ok = True
+
+                            if not aliasing_ok:
                                 prints('WARNING: Family aliasing not fully supported.')
-                                prints('\tDeclared family: %s not in actual families: %s'
+                                prints('\tDeclared family: %r not in actual families: %r'
                                         % (family, families))
                             else:
                                 prints('Loaded embedded font:', repr(family))

From 55b9a96fd8e0d0f06a55174388534b7a2e3bfad9 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Wed, 23 Jun 2010 21:31:01 +0100
Subject: [PATCH 18/31] Device collections editor, new rename collections job,
 changes to tag_list_edit so it could be reused.

---
 src/calibre/devices/prs505/driver.py        |  8 ++-
 src/calibre/devices/prs505/sony_cache.py    |  9 ++-
 src/calibre/devices/usbms/books.py          |  7 +++
 src/calibre/gui2/actions.py                 | 16 ++++++
 src/calibre/gui2/device.py                  | 15 +++++
 src/calibre/gui2/dialogs/tag_list_editor.py | 64 +++------------------
 src/calibre/gui2/init.py                    | 14 ++++-
 src/calibre/gui2/library/models.py          | 51 +++++++++++++---
 src/calibre/gui2/library/views.py           |  9 ++-
 src/calibre/gui2/tag_view.py                | 30 ++++++++--
 src/calibre/manual/faq.rst                  | 62 +++++++++++---------
 11 files changed, 181 insertions(+), 104 deletions(-)

diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index 023416bdf2..fc92f11dc3 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -99,7 +99,7 @@ class PRS505(USBMS):
         if self._card_b_prefix is not None:
             if not write_cache(self._card_b_prefix):
                 self._card_b_prefix = None
-
+        self.booklist_class.rebuild_collections = self.rebuild_collections
 
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
@@ -156,4 +156,10 @@ class PRS505(USBMS):
         USBMS.sync_booklists(self, booklists, end_session=end_session)
         debug_print('PRS505: finished sync_booklists')
 
+    def rebuild_collections(self, booklist, oncard):
+        debug_print('PRS505: started rebuild_collections')
+        c = self.initialize_XML_cache()
+        c.rebuild_collections(booklist, {'carda':1, 'cardb':2}.get(oncard, 0))
+        c.write()
+        debug_print('PRS505: finished rebuild_collections')
 
diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py
index a704824a3f..9d853fbbd6 100644
--- a/src/calibre/devices/prs505/sony_cache.py
+++ b/src/calibre/devices/prs505/sony_cache.py
@@ -61,8 +61,7 @@ class XMLCache(object):
 
     def __init__(self, paths, prefixes, use_author_sort):
         if DEBUG:
-            debug_print('Building XMLCache...')
-            pprint(paths)
+            debug_print('Building XMLCache...', paths)
         self.paths = paths
         self.prefixes = prefixes
         self.use_author_sort = use_author_sort
@@ -347,6 +346,12 @@ class XMLCache(object):
         self.fix_ids()
         debug_print('Finished update XML from JSON')
 
+    def rebuild_collections(self, booklist, bl_index):
+        if bl_index not in self.record_roots:
+            return
+        root = self.record_roots[bl_index]
+        self.update_playlists(bl_index, root, booklist, [])
+
     def update_playlists(self, bl_index, root, booklist, collections_attributes):
         debug_print('Starting update_playlists', collections_attributes)
         collections = booklist.get_collections(collections_attributes)
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index be154f35c1..fe34ea4e53 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -167,3 +167,10 @@ class CollectionsBookList(BookList):
                 books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
         return collections
 
+    def rebuild_collections(self, booklist, oncard):
+        '''
+        For each book in the booklist for the card oncard, remove it from all
+        its current collections, then add it to the collections specified in
+        device_collections.
+        '''
+        pass
diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py
index a3f8442200..d2049d3925 100644
--- a/src/calibre/gui2/actions.py
+++ b/src/calibre/gui2/actions.py
@@ -21,6 +21,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
+from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \
     fetch_scheduled_recipe, generate_catalog
 from calibre.constants import preferred_encoding, filesystem_encoding, \
@@ -831,6 +832,21 @@ class EditMetadataAction(object): # {{{
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
         # }}}
 
+    def edit_device_collections(self, view):
+        model = view.model()
+        result = model.get_collections_with_ids()
+        compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        d = TagListEditor(self, tag_to_match=None, data=result, compare=compare)
+        d.exec_()
+        if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            for text in to_rename:
+                model.rename_collection(old_id=to_rename[text], new_name=unicode(text))
+            for item in to_delete:
+                model.delete_collection_using_id(item)
+            self.upload_collections(model.db, view=view)
+
     # }}}
 
 class SaveToDiskAction(object): # {{{
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 836105aba9..f9ed9cd7f5 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -294,6 +294,11 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
+    def upload_collections(self, done, booklist, on_card):
+        return self.create_job(booklist.rebuild_collections, done,
+                               args=[booklist, on_card],
+                        description=_('Send collections to device'))
+
     def _upload_books(self, files, names, on_card=None, metadata=None):
         '''Upload books to device: '''
         return self.device.upload_books(files, names, on_card,
@@ -1234,6 +1239,16 @@ class DeviceMixin(object): # {{{
         self.card_a_view.reset()
         self.card_b_view.reset()
 
+    def _upload_collections(self, job, view):
+        view.reset()
+
+    def upload_collections(self, booklist, view):
+        on_card = 'carda' if self.stack.currentIndex() == 2 else \
+                  'cardb' if self.stack.currentIndex() == 3 else \
+                   None
+        done = partial(self._upload_collections, view=view)
+        return self.device_manager.upload_collections(done, booklist, on_card)
+
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''
         Upload books to device.
diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 73e81e2d99..07e85829e1 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -1,54 +1,34 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from functools import partial
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
-from calibre.ebooks.metadata import title_sort
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, db, tag_to_match, category, data, compare):
+    def __init__(self, window, tag_to_match, data, compare):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
 
         self.to_rename = {}
         self.to_delete = []
-        self.db = db
         self.all_tags = {}
-        self.category = category
-        if category == 'tags':
-            result = db.get_tags_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        elif category == 'series':
-            result = db.get_series_with_ids()
-            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
-        elif category == 'publisher':
-            result = db.get_publishers_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        else: # should be a custom field
-            self.cc_label = None
-            if category in db.field_metadata:
-                self.cc_label = db.field_metadata[category]['label']
-                result = self.db.get_custom_items_with_ids(label=self.cc_label)
-            else:
-                result = []
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
 
-        for k,v in result:
+        for k,v in data:
             self.all_tags[v] = k
         for tag in sorted(self.all_tags.keys(), cmp=compare):
             item = QListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)
 
-        items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
-        if len(items) == 1:
-            self.available_tags.setCurrentItem(items[0])
+        if tag_to_match is not None:
+            items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
+            if len(items) == 1:
+                self.available_tags.setCurrentItem(items[0])
 
         self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
         self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
@@ -62,11 +42,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 item.setText(self.item_before_editing.text())
                 return
         if item.text() != self.item_before_editing.text():
-            if item.text() in self.all_tags.keys() or item.text() in self.to_rename.keys():
-                error_dialog(self, _('Item already used'),
-                             _('The item %s is already used.')%(item.text())).exec_()
-                item.setText(self.item_before_editing.text())
-                return
             (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
             self.to_rename[item.text()] = id
 
@@ -100,29 +75,4 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             self.available_tags.takeItem(self.available_tags.row(item))
 
     def accept(self):
-        rename_func = None
-        if self.category == 'tags':
-            rename_func = self.db.rename_tag
-            delete_func = self.db.delete_tag_using_id
-        elif self.category == 'series':
-            rename_func = self.db.rename_series
-            delete_func = self.db.delete_series_using_id
-        elif self.category == 'publisher':
-            rename_func = self.db.rename_publisher
-            delete_func = self.db.delete_publisher_using_id
-        else:
-            rename_func = partial(self.db.rename_custom_item, label=self.cc_label)
-            delete_func = partial(self.db.delete_custom_item_using_id, label=self.cc_label)
-
-        work_done = False
-        if rename_func:
-            for text in self.to_rename:
-                work_done = True
-                rename_func(id=self.to_rename[text], new_name=unicode(text))
-            for item in self.to_delete:
-                work_done = True
-                delete_func(item)
-        if not work_done:
-            QDialog.reject(self)
-        else:
-            QDialog.accept(self)
+        QDialog.accept(self)
diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index b334808d9b..6df1062e68 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -226,17 +226,22 @@ class LibraryViewMixin(object): # {{{
                                         self.action_show_book_details,
                                         self.action_del,
                                         add_to_library = None,
+                                        edit_device_collections=None,
                                         similar_menu=similar_menu)
         add_to_library = (_('Add books to library'), self.add_books_from_device)
+        edit_device_collections = (_('Manage collections'), self.edit_device_collections)
         self.memory_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
         self.card_a_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
         self.card_b_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
 
         self.library_view.files_dropped.connect(self.files_dropped, type=Qt.QueuedConnection)
         for func, args in [
@@ -249,8 +254,11 @@ class LibraryViewMixin(object): # {{{
                 getattr(view, func)(*args)
 
         self.memory_view.connect_dirtied_signal(self.upload_booklists)
+        self.memory_view.connect_upload_collections_signal(self.upload_collections)
         self.card_a_view.connect_dirtied_signal(self.upload_booklists)
+        self.card_a_view.connect_upload_collections_signal(self.upload_collections)
         self.card_b_view.connect_dirtied_signal(self.upload_booklists)
+        self.card_b_view.connect_upload_collections_signal(self.upload_collections)
 
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index 008f024aae..de3f9bad1f 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -857,6 +857,7 @@ class OnDeviceSearch(SearchQueryParser): # {{{
 class DeviceBooksModel(BooksModel): # {{{
 
     booklist_dirtied = pyqtSignal()
+    upload_collections = pyqtSignal(object)
 
     def __init__(self, parent):
         BooksModel.__init__(self, parent)
@@ -977,8 +978,8 @@ class DeviceBooksModel(BooksModel): # {{{
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(self.db[x].device_collections)
-            y = ','.join(self.db[y].device_collections)
+            x = ','.join(getattr(self.db[x], 'device_collections', [])).lower()
+            y = ','.join(getattr(self.db[y], 'device_collections', [])).lower()
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1026,6 +1027,9 @@ class DeviceBooksModel(BooksModel): # {{{
     def set_database(self, db):
         self.custom_columns = {}
         self.db = db
+        for book in db:
+            if book.device_collections is not None:
+                book.device_collections.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
         self.map = list(range(0, len(db)))
 
     def current_changed(self, current, previous):
@@ -1079,6 +1083,36 @@ class DeviceBooksModel(BooksModel): # {{{
                 res.append((r,b))
         return res
 
+    def get_collections_with_ids(self):
+        collections = set()
+        for book in self.db:
+            if book.device_collections is not None:
+                collections.update(set(book.device_collections))
+        self.collections = []
+        result = []
+        for i,collection in enumerate(collections):
+            result.append((i, collection))
+            self.collections.append(collection)
+        return result
+
+    def rename_collection(self, old_id, new_name):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+                if new_name not in book.device_collections:
+                    book.device_collections.append(new_name)
+
+    def delete_collection_using_id(self, old_id):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+
     def indices(self, rows):
         '''
         Return indices into underlying database from rows
@@ -1109,7 +1143,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
-                    return QVariant(', '.join(sorted(tags, key=str.lower)))
+                    return QVariant(', '.join(tags))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
                 return QVariant(_('Marked for deletion'))
@@ -1151,14 +1185,17 @@ class DeviceBooksModel(BooksModel): # {{{
                 return False
             val = unicode(value.toString()).strip()
             idx = self.map[row]
+            if cname == 'collections':
+                tags = [i.strip() for i in val.split(',')]
+                tags = [t for t in tags if t]
+                self.db[idx].device_collections = tags
+                self.upload_collections.emit(self.db)
+                return True
+
             if cname == 'title' :
                 self.db[idx].title = val
             elif cname == 'authors':
                 self.db[idx].authors = string_to_authors(val)
-            elif cname == 'collections':
-                tags = [i.strip() for i in val.split(',')]
-                tags = [t for t in tags if t]
-                self.db[idx].device_collections = tags
             self.dataChanged.emit(index, index)
             self.booklist_dirtied.emit()
             done = True
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index 8245c2d188..c0d6792399 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -371,7 +371,8 @@ class BooksView(QTableView): # {{{
     # Context Menu {{{
     def set_context_menu(self, edit_metadata, send_to_device, convert, view,
                          save, open_folder, book_details, delete,
-                         similar_menu=None, add_to_library=None):
+                         similar_menu=None, add_to_library=None,
+                         edit_device_collections=None):
         self.setContextMenuPolicy(Qt.DefaultContextMenu)
         self.context_menu = QMenu(self)
         if edit_metadata is not None:
@@ -393,6 +394,9 @@ class BooksView(QTableView): # {{{
         if add_to_library is not None:
             func = partial(add_to_library[1], view=self)
             self.context_menu.addAction(add_to_library[0], func)
+        if edit_device_collections is not None:
+            func = partial(edit_device_collections[1], view=self)
+            self.context_menu.addAction(edit_device_collections[0], func)
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
@@ -505,6 +509,9 @@ class DeviceBooksView(BooksView): # {{{
     def connect_dirtied_signal(self, slot):
         self._model.booklist_dirtied.connect(slot)
 
+    def connect_upload_collections_signal(self, func):
+        self._model.upload_collections.connect(partial(func, view=self))
+
     def dropEvent(self, *args):
         error_dialog(self, _('Not allowed'),
         _('Dropping onto a device is not supported. First add the book to the calibre library.')).exec_()
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 6b4e8dbd88..140b1e1e52 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -692,18 +692,38 @@ class TagBrowserMixin(object): # {{{
             result = db.get_publishers_with_ids()
             compare = (lambda x,y:cmp(x.lower(), y.lower()))
         else: # should be a custom field
-            self.cc_label = None
+            cc_label = None
             if category in db.field_metadata:
-                self.cc_label = db.field_metadata[category]['label']
-                result = self.db.get_custom_items_with_ids(label=self.cc_label)
+                cc_label = db.field_metadata[category]['label']
+                result = self.db.get_custom_items_with_ids(label=cc_label)
             else:
                 result = []
             compare = (lambda x,y:cmp(x.lower(), y.lower()))
 
-        d = TagListEditor(self, db=db, tag_to_match=tag, category=category,
-                          data=result, compare=compare)
+        d = TagListEditor(self, tag_to_match=tag, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            rename_func = None
+            if category == 'tags':
+                rename_func = db.rename_tag
+                delete_func = db.delete_tag_using_id
+            elif category == 'series':
+                rename_func = db.rename_series
+                delete_func = db.delete_series_using_id
+            elif category == 'publisher':
+                rename_func = db.rename_publisher
+                delete_func = db.delete_publisher_using_id
+            else:
+                rename_func = partial(db.rename_custom_item, label=cc_label)
+                delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
+            if rename_func:
+                for text in to_rename:
+                    rename_func(old_id=to_rename[text], new_name=unicode(text))
+                for item in to_delete:
+                    delete_func(item)
+
             # Clean up everything, as information could have changed for many books.
             self.library_view.model().refresh()
             self.tags_view.set_new_model()
diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 974cbf27fc..67353fa34b 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -7,7 +7,7 @@ Frequently Asked Questions
 
 .. contents:: Contents
   :depth: 1
-  :local: 
+  :local:
 
 E-book Format Conversion
 -------------------------
@@ -30,7 +30,7 @@ It can convert every input format in the following list, to every output format.
 
 What are the best source formats to convert?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF 
+In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF
 
 Why does the PDF conversion lose some images/tables?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -40,7 +40,7 @@ are also represented as vector diagrams, thus they cannot be extracted.
 How do I convert a collection of HTML files in a specific order?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
- 
+
    <html>
       <body>
         <h1>Table of Contents</h1>
@@ -60,16 +60,16 @@ Then just add this HTML file to the GUI and use the convert button to create you
 
 How do I convert my file containing non-English characters, or smart quotes?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-There are two aspects to this problem: 
+There are two aspects to this problem:
   1. Knowing the encoding of the source file: |app| tries to guess what character encoding your source files use, but often, this is impossible, so you need to tell it what encoding to use. This can be done in the GUI via the :guilabel:`Input character encoding` field in the :guilabel:`Look & Feel` section. The command-line tools all have an :option:`--input-encoding` option.
-  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8). 
-  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader. 
+  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8).
+  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader.
 
 
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features: 
-  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_ 
+ You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features:
+  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_
 
 
 Device Integration
@@ -95,7 +95,7 @@ We just need some information from you:
     device supports SD cards, insert them. Then connect your device. In calibre go to Preferences->Advanced
     and click the "Debug device detection" button. This will create some debug output. Copy it to a file
     and repeat the process, this time with your device disconnected.
-  * Send both the above outputs to us with the other information and we will write a device driver for your 
+  * Send both the above outputs to us with the other information and we will write a device driver for your
     device.
 
 Once you send us the output for a particular operating system, support for the device in that operating system
@@ -104,14 +104,20 @@ will appear in the next release of |app|.
 How does |app| manage collections on my SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When you send a book to the device, |app| will create collections based on the metadata for that book. By default, collections are created
-from tags and series. You can control what metadata is used by going to Preferences->Plugins->Device Interface plugins and customizing
-the SONY device interface plugin. 
+When |app| connects with the device, it retrieves all collections for the books on the device. The collections
+of which books are members are shown on the device view.
 
-You can edit collections on the device in the device view in |app| by double clicking or right clicking on the collections field.
+When you send a book to the device, |app| will if necessary create new collections based on the metadata for
+that book, then add the book to the collections. By default, collections are created from tags and series. You
+can control what metadata is used by going to Preferences->Plugins->Device Interface plugins and customizing
+the SONY device interface plugin.
 
-|app| will not delete already existing collections on your device. To ensure that the collections are based only on current |app| metadata,
-delete and resend the books to the device.
+|app| will not delete already existing collections for a book on your device when you resend the book to the
+device. To ensure that the collections are based only on current |app| metadata, first delete the books from
+the device, and then resend the books.
+
+You can edit collections on the device in the device view in |app| by double clicking or right clicking on the
+collections field. This is the only way to remove a book from a collection.
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -141,7 +147,7 @@ simplest is to simply re-name the executable file that launches the library prog
 Can I use the collections feature of the SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| has full support for collections. When you add tags to a book's metadata, those tags are turned into collections when you upload the book to the SONY reader. Also, the series information is automatically
-turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does. 
+turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does.
 
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -150,7 +156,7 @@ The easiest way to browse your |app| collection on your Apple device (iPad/iPhon
 
 First perform the following steps in |app|
 
-  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General) 
+  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General)
   * Set the output profile to iPad (this will work for iPhone/iPods as well), under Preferences->Conversion->Page Setup
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
@@ -182,7 +188,7 @@ Can I access my |app| books using the web browser in my Kindle or other reading
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 |app| has a *Content Server* that exports the books in |app| as a web page. You can turn it on under
-Preferences->Content Server. Then just point the web browser on your device to the computer running 
+Preferences->Content Server. Then just point the web browser on your device to the computer running
 the Content Server and you will be able to browse your book collection. For example, if the computer running
 the server has IP address 63.45.128.5, in the browser, you would type::
 
@@ -201,8 +207,8 @@ The most likely cause of this is your antivirus program. Try temporarily disabli
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests:: 
-    
+|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests::
+
     grep SYSFS_DEPRECATED /boot/config-`uname -r`
 
 You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``.
@@ -238,7 +244,7 @@ Now this makes it very easy to find for example all science fiction books by Isa
 
     ReadStatus -> Genre -> Author -> Series
 
-In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search. 
+In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search.
 
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -246,7 +252,7 @@ Why doesn't |app| have a column for foo?
 
 How do I move my |app| library from one computer to another?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. 
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder.
 
 Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to Preferences->Advanced and click the Check database integrity button. It will warn you about missing files, if any, which you should then transfer by hand.
 
@@ -257,11 +263,11 @@ Content From The Web
   :depth: 1
   :local:
 
-My downloaded news content causes the reader to reset. 
+My downloaded news content causes the reader to reset.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 This is a bug in the SONY firmware. The problem can be mitigated by switching the output format to EPUB
-in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software 
-to transfer the files to the reader. The SONY software pre-paginates the LRF file, 
+in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software
+to transfer the files to the reader. The SONY software pre-paginates the LRF file,
 thereby reducing the number of resets.
 
 I obtained a recipe for a news site as a .py file from somewhere, how do I use it?
@@ -296,7 +302,7 @@ Take your pick:
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts founf on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory. 
+|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts founf on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
 
 |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -340,8 +346,8 @@ Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. Thi
 
 I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-You have two choices: 
- 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_. 
+You have two choices:
+ 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
  2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first) and hopefully I will find the time to implement your feature.
 
 Can I include |app| on a CD to be distributed with my product/magazine?

From 7ea679768ed07d0ea09b30e5e48d3dad23ae4805 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 14:54:54 -0600
Subject: [PATCH 19/31] HTML Input: Handle absolute paths in resource links on
 windows correctly. Fixes #3031 (HTML input: Improper handling of local URLs)

---
 src/calibre/ebooks/html/input.py | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py
index 6108aa329d..229d71e574 100644
--- a/src/calibre/ebooks/html/input.py
+++ b/src/calibre/ebooks/html/input.py
@@ -20,7 +20,7 @@ from itertools import izip
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
-from calibre.constants import islinux, isfreebsd
+from calibre.constants import islinux, isfreebsd, iswindows
 from calibre import unicode_path
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
@@ -32,9 +32,14 @@ class Link(object):
 
     @classmethod
     def url_to_local_path(cls, url, base):
-        path = urlunparse(('', '', url.path, url.params, url.query, ''))
+        path = url.path
+        isabs = False
+        if iswindows and path.startswith('/'):
+            path = path[1:]
+            isabs = True
+        path = urlunparse(('', '', path, url.params, url.query, ''))
         path = unquote(path)
-        if os.path.isabs(path):
+        if isabs or os.path.isabs(path):
             return path
         return os.path.abspath(os.path.join(base, path))
 
@@ -411,10 +416,23 @@ class HTMLInput(InputFormatPlugin):
 
 
     def resource_adder(self, link_, base=None):
-        link = self.urlnormalize(link_)
-        link, frag = self.urldefrag(link)
-        link = unquote(link).replace('/', os.sep)
-        if not link.strip():
+        if not isinstance(link_, unicode):
+            try:
+                link_ = link_.decode('utf-8', 'error')
+            except:
+                self.log.warn('Failed to decode link %r. Ignoring'%link_)
+                return link_
+        try:
+            l = Link(link_, base if base else os.path.getcwdu())
+        except:
+            self.log.exception('Failed to process link: %r'%link_)
+            return link_
+        if l.path is None:
+            # Not a local resource
+            return link_
+        link = l.path.replace('/', os.sep).strip()
+        frag = l.fragment
+        if not link:
             return link_
         try:
             if base and not os.path.isabs(link):

From ffb9002c325f708abc0a5b6f71523151003b7c7e Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 23 Jun 2010 21:39:33 -0600
Subject: [PATCH 20/31] HTML Input: Handle @import directives in linked css
 files. Fixes #5135 (style import error during HTML conversion with
 ebook-convert.exe)

---
 src/calibre/ebooks/conversion/preprocess.py | 16 ++++++++--
 src/calibre/ebooks/html/input.py            | 35 ++++++++++++++++-----
 src/calibre/ebooks/oeb/base.py              | 33 +++++++++++++++++--
 3 files changed, 71 insertions(+), 13 deletions(-)

diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py
index 7a7f362169..4734f4f5c1 100644
--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@@ -107,9 +107,21 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
 
-    def __call__(self, data):
+    def __call__(self, data, add_namespace=False):
+        from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
-        return data
+        if not add_namespace:
+            return data
+        ans, namespaced = [], False
+        for line in data.splitlines():
+            ll = line.lstrip()
+            if not (namespaced or ll.startswith('@import') or
+                        ll.startswith('@charset')):
+                ans.append(XHTML_CSS_NAMESPACE.strip())
+                namespaced = True
+            ans.append(line)
+
+        return u'\n'.join(ans)
 
 class HTMLPreProcessor(object):
 
diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py
index 229d71e574..73fd020d7b 100644
--- a/src/calibre/ebooks/html/input.py
+++ b/src/calibre/ebooks/html/input.py
@@ -312,6 +312,7 @@ class HTMLInput(InputFormatPlugin):
             xpath
         from calibre import guess_type
         import cssutils
+        self.OEB_STYLES = OEB_STYLES
         oeb = create_oebbook(log, None, opts, self,
                 encoding=opts.input_encoding, populate=False)
         self.oeb = oeb
@@ -376,7 +377,7 @@ class HTMLInput(InputFormatPlugin):
             rewrite_links(item.data, partial(self.resource_adder, base=dpath))
 
         for item in oeb.manifest.values():
-            if item.media_type in OEB_STYLES:
+            if item.media_type in self.OEB_STYLES:
                 dpath = None
                 for path, href in self.added_resources.items():
                     if href == item.href:
@@ -414,25 +415,30 @@ class HTMLInput(InputFormatPlugin):
         oeb.container = DirContainer(os.getcwdu(), oeb.log)
         return oeb
 
-
-    def resource_adder(self, link_, base=None):
+    def link_to_local_path(self, link_, base=None):
         if not isinstance(link_, unicode):
             try:
                 link_ = link_.decode('utf-8', 'error')
             except:
                 self.log.warn('Failed to decode link %r. Ignoring'%link_)
-                return link_
+                return None, None
         try:
-            l = Link(link_, base if base else os.path.getcwdu())
+            l = Link(link_, base if base else os.getcwdu())
         except:
             self.log.exception('Failed to process link: %r'%link_)
-            return link_
+            return None, None
         if l.path is None:
             # Not a local resource
-            return link_
+            return None, None
         link = l.path.replace('/', os.sep).strip()
         frag = l.fragment
         if not link:
+            return None, None
+        return link, frag
+
+    def resource_adder(self, link_, base=None):
+        link, frag = self.link_to_local_path(link_, base=base)
+        if link is None:
             return link_
         try:
             if base and not os.path.isabs(link):
@@ -460,6 +466,9 @@ class HTMLInput(InputFormatPlugin):
 
             item = self.oeb.manifest.add(id, href, media_type)
             item.html_input_href = bhref
+            if guessed in self.OEB_STYLES:
+                item.override_css_fetch = partial(
+                        self.css_import_handler, os.path.dirname(link))
             item.data
             self.added_resources[link] = href
 
@@ -468,7 +477,17 @@ class HTMLInput(InputFormatPlugin):
             nlink = '#'.join((nlink, frag))
         return nlink
 
-
+    def css_import_handler(self, base, href):
+        link, frag = self.link_to_local_path(href, base=base)
+        if link is None or not os.access(link, os.R_OK) or os.path.isdir(link):
+            return (None, None)
+        try:
+            raw = open(link, 'rb').read().decode('utf-8', 'replace')
+            raw = self.oeb.css_preprocessor(raw, add_namespace=True)
+        except:
+            self.log.exception('Failed to read CSS file: %r'%link)
+            return (None, None)
+        return (None, raw)
 
 
 
diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py
index 76e2cef3bb..54549ac415 100644
--- a/src/calibre/ebooks/oeb/base.py
+++ b/src/calibre/ebooks/oeb/base.py
@@ -17,6 +17,7 @@ from urlparse import urljoin
 
 from lxml import etree, html
 from cssutils import CSSParser
+from cssutils.css import CSSRule
 
 import calibre
 from calibre.constants import filesystem_encoding
@@ -762,6 +763,7 @@ class Manifest(object):
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
+            self.override_css_fetch = None
             self.spine_position = None
             self.linear = True
             if loader is None and data is None:
@@ -982,15 +984,40 @@ class Manifest(object):
 
 
         def _parse_css(self, data):
+
+            def get_style_rules_from_import(import_rule):
+                ans = []
+                if not import_rule.styleSheet:
+                    return ans
+                rules = import_rule.styleSheet.cssRules
+                for rule in rules:
+                    if rule.type == CSSRule.IMPORT_RULE:
+                        ans.extend(get_style_rules_from_import(rule))
+                    elif rule.type in (CSSRule.FONT_FACE_RULE,
+                            CSSRule.STYLE_RULE):
+                        ans.append(rule)
+                return ans
+
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
-            data = self.oeb.css_preprocessor(data)
-            data = XHTML_CSS_NAMESPACE + data
+            data = self.oeb.css_preprocessor(data, add_namespace=True)
             parser = CSSParser(loglevel=logging.WARNING,
-                               fetcher=self._fetch_css,
+                               fetcher=self.override_css_fetch or self._fetch_css,
                                log=_css_logger)
             data = parser.parseString(data, href=self.href)
             data.namespaces['h'] = XHTML_NS
+            import_rules = list(data.cssRules.rulesOfType(CSSRule.IMPORT_RULE))
+            rules_to_append = []
+            insert_index = None
+            for r in data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
+                insert_index = data.cssRules.index(r)
+                break
+            for rule in import_rules:
+                rules_to_append.extend(get_style_rules_from_import(rule))
+            for r in reversed(rules_to_append):
+                data.insertRule(r, index=insert_index)
+            for rule in import_rules:
+                data.deleteRule(rule)
             return data
 
         def _fetch_css(self, path):

From 46a9d48b1d27a27d2c23de90f622df89af1c80b7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 08:41:37 +0100
Subject: [PATCH 21/31] Small corrections to collection editing & collections
 sorting on display

---
 src/calibre/devices/prs505/driver.py     |  2 +-
 src/calibre/devices/prs505/sony_cache.py |  3 ++-
 src/calibre/gui2/actions.py              |  5 +++--
 src/calibre/gui2/device.py               | 12 ++++--------
 src/calibre/gui2/init.py                 | 20 +++++++++++++++-----
 src/calibre/gui2/library/models.py       |  9 ++++-----
 src/calibre/gui2/library/views.py        |  4 ++--
 7 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py
index fc92f11dc3..c55936be2d 100644
--- a/src/calibre/devices/prs505/driver.py
+++ b/src/calibre/devices/prs505/driver.py
@@ -157,7 +157,7 @@ class PRS505(USBMS):
         debug_print('PRS505: finished sync_booklists')
 
     def rebuild_collections(self, booklist, oncard):
-        debug_print('PRS505: started rebuild_collections')
+        debug_print('PRS505: started rebuild_collections on card', oncard)
         c = self.initialize_XML_cache()
         c.rebuild_collections(booklist, {'carda':1, 'cardb':2}.get(oncard, 0))
         c.write()
diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py
index 9d853fbbd6..289147482c 100644
--- a/src/calibre/devices/prs505/sony_cache.py
+++ b/src/calibre/devices/prs505/sony_cache.py
@@ -351,9 +351,10 @@ class XMLCache(object):
             return
         root = self.record_roots[bl_index]
         self.update_playlists(bl_index, root, booklist, [])
+        self.fix_ids()
 
     def update_playlists(self, bl_index, root, booklist, collections_attributes):
-        debug_print('Starting update_playlists', collections_attributes)
+        debug_print('Starting update_playlists', collections_attributes, bl_index)
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py
index d2049d3925..f3f6ee604c 100644
--- a/src/calibre/gui2/actions.py
+++ b/src/calibre/gui2/actions.py
@@ -832,7 +832,7 @@ class EditMetadataAction(object): # {{{
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
         # }}}
 
-    def edit_device_collections(self, view):
+    def edit_device_collections(self, view, oncard=None):
         model = view.model()
         result = model.get_collections_with_ids()
         compare = (lambda x,y:cmp(x.lower(), y.lower()))
@@ -845,7 +845,8 @@ class EditMetadataAction(object): # {{{
                 model.rename_collection(old_id=to_rename[text], new_name=unicode(text))
             for item in to_delete:
                 model.delete_collection_using_id(item)
-            self.upload_collections(model.db, view=view)
+            self.upload_collections(model.db, view=view, oncard=oncard)
+            view.reset()
 
     # }}}
 
diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index e956eca562..6be50cf293 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1239,17 +1239,13 @@ class DeviceMixin(object): # {{{
         self.card_a_view.reset()
         self.card_b_view.reset()
 
-    def _upload_collections(self, job, view):
+    def _upload_collections(self, job):
         if job.failed:
             self.device_job_exception(job)
-        view.reset()
 
-    def upload_collections(self, booklist, view):
-        on_card = 'carda' if self.stack.currentIndex() == 2 else \
-                  'cardb' if self.stack.currentIndex() == 3 else \
-                   None
-        done = partial(self._upload_collections, view=view)
-        return self.device_manager.upload_collections(done, booklist, on_card)
+    def upload_collections(self, booklist, view=None, oncard=None):
+        return self.device_manager.upload_collections(self._upload_collections,
+                                                       booklist, oncard)
 
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''
diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index 6df1062e68..efbe32a04e 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -229,15 +229,23 @@ class LibraryViewMixin(object): # {{{
                                         edit_device_collections=None,
                                         similar_menu=similar_menu)
         add_to_library = (_('Add books to library'), self.add_books_from_device)
-        edit_device_collections = (_('Manage collections'), self.edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard=None))
         self.memory_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
                 add_to_library=add_to_library,
                 edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='carda'))
         self.card_a_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
                 add_to_library=add_to_library,
                 edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='cardb'))
         self.card_b_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
                 add_to_library=add_to_library,
@@ -254,12 +262,14 @@ class LibraryViewMixin(object): # {{{
                 getattr(view, func)(*args)
 
         self.memory_view.connect_dirtied_signal(self.upload_booklists)
-        self.memory_view.connect_upload_collections_signal(self.upload_collections)
+        self.memory_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard=None)
         self.card_a_view.connect_dirtied_signal(self.upload_booklists)
-        self.card_a_view.connect_upload_collections_signal(self.upload_collections)
+        self.card_a_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='carda')
         self.card_b_view.connect_dirtied_signal(self.upload_booklists)
-        self.card_b_view.connect_upload_collections_signal(self.upload_collections)
-
+        self.card_b_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='cardb')
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
         self.library_view.set_database(db)
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index de3f9bad1f..d8b85fcd0d 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -978,8 +978,8 @@ class DeviceBooksModel(BooksModel): # {{{
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(getattr(self.db[x], 'device_collections', [])).lower()
-            y = ','.join(getattr(self.db[y], 'device_collections', [])).lower()
+            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []))).lower()
+            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []))).lower()
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1027,9 +1027,6 @@ class DeviceBooksModel(BooksModel): # {{{
     def set_database(self, db):
         self.custom_columns = {}
         self.db = db
-        for book in db:
-            if book.device_collections is not None:
-                book.device_collections.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
         self.map = list(range(0, len(db)))
 
     def current_changed(self, current, previous):
@@ -1143,6 +1140,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
+                    tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
                     return QVariant(', '.join(tags))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
@@ -1189,6 +1187,7 @@ class DeviceBooksModel(BooksModel): # {{{
                 tags = [i.strip() for i in val.split(',')]
                 tags = [t for t in tags if t]
                 self.db[idx].device_collections = tags
+                self.dataChanged.emit(index, index)
                 self.upload_collections.emit(self.db)
                 return True
 
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index c0d6792399..463dccbc1f 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -509,8 +509,8 @@ class DeviceBooksView(BooksView): # {{{
     def connect_dirtied_signal(self, slot):
         self._model.booklist_dirtied.connect(slot)
 
-    def connect_upload_collections_signal(self, func):
-        self._model.upload_collections.connect(partial(func, view=self))
+    def connect_upload_collections_signal(self, func=None, oncard=None):
+        self._model.upload_collections.connect(partial(func, view=self, oncard=oncard))
 
     def dropEvent(self, *args):
         error_dialog(self, _('Not allowed'),

From a7e20ef5170c94076084b96a8ed75e21bc2a2963 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 10:37:58 +0100
Subject: [PATCH 22/31] Hide edit_collections on context menu when device does
 not support collections

---
 src/calibre/gui2/library/views.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index 463dccbc1f..19daa1353c 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -396,7 +396,8 @@ class BooksView(QTableView): # {{{
             self.context_menu.addAction(add_to_library[0], func)
         if edit_device_collections is not None:
             func = partial(edit_device_collections[1], view=self)
-            self.context_menu.addAction(edit_device_collections[0], func)
+            self.edit_collections_menu = \
+                self.context_menu.addAction(edit_device_collections[0], func)
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
@@ -498,6 +499,11 @@ class DeviceBooksView(BooksView): # {{{
         self.setDragDropMode(self.NoDragDrop)
         self.setAcceptDrops(False)
 
+    def contextMenuEvent(self, event):
+        self.edit_collections_menu.setVisible(self._model.db.supports_collections())
+        self.context_menu.popup(event.globalPos())
+        event.accept()
+
     def set_database(self, db):
         self._model.set_database(db)
         self.restore_state()

From ee97153ce4e0d9c85fd15f874d3e94d6b487e08e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 14:08:01 +0100
Subject: [PATCH 23/31] Add an option to not preserve user collections, but
 instead have them controlled by calibre metadata. Make the tags_list_editor
 show the old name when renaming

---
 src/calibre/devices/prs505/sony_cache.py    | 17 +++++------
 src/calibre/devices/usbms/books.py          |  7 +++--
 src/calibre/ebooks/metadata/__init__.py     |  7 +++--
 src/calibre/gui2/dialogs/config/add_save.py |  2 ++
 src/calibre/gui2/dialogs/config/add_save.ui | 28 ++++++++++++++++--
 src/calibre/gui2/dialogs/tag_list_editor.py | 32 ++++++++++++++++++++-
 src/calibre/gui2/library/models.py          |  7 +++--
 src/calibre/gui2/library/views.py           |  6 ++--
 src/calibre/gui2/ui.py                      |  2 ++
 src/calibre/utils/config.py                 |  2 ++
 10 files changed, 88 insertions(+), 22 deletions(-)

diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py
index 289147482c..9bd005d71c 100644
--- a/src/calibre/devices/prs505/sony_cache.py
+++ b/src/calibre/devices/prs505/sony_cache.py
@@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, time
-from pprint import pprint
 from base64 import b64decode
 from uuid import uuid4
 from lxml import etree
@@ -179,8 +178,8 @@ class XMLCache(object):
                 path = record.get('path', None)
                 if path:
                     if path not in playlist_map:
-                        playlist_map[path] = set()
-                    playlist_map[path].add(title)
+                        playlist_map[path] = []
+                    playlist_map[path].append(title)
         debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
         return playlist_map
 
@@ -309,14 +308,14 @@ class XMLCache(object):
                             book.thumbnail = raw
                             break
                     break
-                book.device_collections = list(playlist_map.get(book.lpath, set()))
+                book.device_collections = playlist_map.get(book.lpath, [])
         debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
-        debug_print('In update. Starting update XML from JSON')
+        debug_print('Starting update', collections_attributes)
         for i, booklist in booklists.items():
             playlist_map = self.build_id_playlist_map(i)
             debug_print('Updating XML Cache:', i)
@@ -332,8 +331,7 @@ class XMLCache(object):
                 # this book
                 if book.device_collections is None:
                     book.device_collections = []
-                book.device_collections = list(set(book.device_collections) |
-                                            playlist_map.get(book.lpath, set()))
+                book.device_collections = playlist_map.get(book.lpath, [])
             self.update_playlists(i, root, booklist, collections_attributes)
         # Update the device collections because update playlist could have added
         # some new ones.
@@ -341,10 +339,9 @@ class XMLCache(object):
         for i, booklist in booklists.items():
             playlist_map = self.build_id_playlist_map(i)
             for book in booklist:
-                book.device_collections = list(set(book.device_collections) |
-                                            playlist_map.get(book.lpath, set()))
+                book.device_collections = playlist_map.get(book.lpath, [])
         self.fix_ids()
-        debug_print('Finished update XML from JSON')
+        debug_print('Finished update')
 
     def rebuild_collections(self, booklist, bl_index):
         if bl_index not in self.record_roots:
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 996ce683c2..7108fa3f00 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -11,6 +11,7 @@ from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
+from calibre.utils.config import prefs
 
 class Book(MetaInformation):
 
@@ -76,7 +77,7 @@ class Book(MetaInformation):
         in C{other} takes precedence, unless the information in C{other} is NULL.
         '''
 
-        MetaInformation.smart_update(self, other)
+        MetaInformation.smart_update(self, other, replace_tags=True)
 
         for attr in self.BOOK_ATTRS:
             if hasattr(other, attr):
@@ -132,7 +133,9 @@ class CollectionsBookList(BookList):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
-        collection_attributes = list(collection_attributes)+['device_collections']
+        collection_attributes = list(collection_attributes)
+        if prefs['preserve_user_collections']:
+            collection_attributes += ['device_collections']
         for attr in collection_attributes:
             attr = attr.strip()
             for book in self:
diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py
index 4d126fda9d..0dbffd5f7f 100644
--- a/src/calibre/ebooks/metadata/__init__.py
+++ b/src/calibre/ebooks/metadata/__init__.py
@@ -268,7 +268,7 @@ class MetaInformation(object):
                   ):
             prints(x, getattr(self, x, 'None'))
 
-    def smart_update(self, mi):
+    def smart_update(self, mi, replace_tags=False):
         '''
         Merge the information in C{mi} into self. In case of conflicts, the information
         in C{mi} takes precedence, unless the information in mi is NULL.
@@ -291,7 +291,10 @@ class MetaInformation(object):
                     setattr(self, attr, val)
 
         if mi.tags:
-            self.tags += mi.tags
+            if replace_tags:
+                self.tags = mi.tags
+            else:
+                self.tags += mi.tags
         self.tags = list(set(self.tags))
 
         if mi.author_sort_map:
diff --git a/src/calibre/gui2/dialogs/config/add_save.py b/src/calibre/gui2/dialogs/config/add_save.py
index aff995d84f..b1f5621f44 100644
--- a/src/calibre/gui2/dialogs/config/add_save.py
+++ b/src/calibre/gui2/dialogs/config/add_save.py
@@ -45,6 +45,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.opt_swap_author_names.setChecked(prefs['swap_author_names'])
         self.opt_add_formats_to_existing.setChecked(prefs['add_formats_to_existing'])
+        self.preserve_user_collections.setChecked(prefs['preserve_user_collections'])
         help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
         self.save_template.initialize('save_to_disk', opts.template, help)
         self.send_template.initialize('send_to_device', opts.send_template, help)
@@ -71,6 +72,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         prefs['filename_pattern'] = pattern
         prefs['swap_author_names'] = bool(self.opt_swap_author_names.isChecked())
         prefs['add_formats_to_existing'] = bool(self.opt_add_formats_to_existing.isChecked())
+        prefs['preserve_user_collections'] = bool(self.preserve_user_collections.isChecked())
 
         return True
 
diff --git a/src/calibre/gui2/dialogs/config/add_save.ui b/src/calibre/gui2/dialogs/config/add_save.ui
index 7fda2dbc7f..a29c0fd2e6 100644
--- a/src/calibre/gui2/dialogs/config/add_save.ui
+++ b/src/calibre/gui2/dialogs/config/add_save.ui
@@ -51,7 +51,7 @@
     <item row="2" column="0" colspan="2">
      <widget class="QCheckBox" name="opt_add_formats_to_existing">
       <property name="toolTip">
-       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added 
+       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
 to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
@@ -179,7 +179,31 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
    </attribute>
    <layout class="QVBoxLayout" name="verticalLayout_2">
     <item>
-     <widget class="QLabel" name="label_4">
+     <widget class="QCheckBox" name="preserve_user_collections">
+      <property name="text">
+       <string>Preserve user collections.</string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_41">
+      <property name="text">
+       <string>If checked, collections will not be deleted even if a book with changed metadata is resent and the collection is not in the book's metadata. In addition, editing collections on the device view will be enabled.</string>
+      </property>
+      <property name="wordWrap">
+       <bool>true</bool>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_42">
+      <property name="text">
+       <string>  </string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_43">
       <property name="text">
        <string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Plugins</string>
       </property>
diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 9cf95f2b62..6d91ed2ee2 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -7,6 +7,36 @@ from PyQt4.QtGui import QDialog, QListWidgetItem
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
 
+class ListWidgetItem(QListWidgetItem):
+
+    def __init__(self, txt):
+        QListWidgetItem.__init__(self, txt)
+        self.old_value = txt
+        self.cur_value = txt
+
+    def data(self, role):
+        if role == Qt.DisplayRole:
+            if self.old_value != self.cur_value:
+                return _('%s (was %s)'%(self.cur_value, self.old_value))
+            else:
+                return self.cur_value
+        elif role == Qt.EditRole:
+            return self.cur_value
+        else:
+            return QListWidgetItem.data(self, role)
+
+    def setData(self, role, data):
+        if role == Qt.EditRole:
+            self.cur_value = data.toString()
+        QListWidgetItem.setData(self, role, data)
+
+    def text(self):
+        return self.cur_value
+
+    def setText(self, txt):
+        self.cur_value = txt
+        QListWidgetItem.setText(txt)
+
 class TagListEditor(QDialog, Ui_TagListEditor):
 
     def __init__(self, window, tag_to_match, data, compare):
@@ -21,7 +51,7 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         for k,v in data:
             self.all_tags[v] = k
         for tag in sorted(self.all_tags.keys(), cmp=compare):
-            item = QListWidgetItem(tag)
+            item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)
 
diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py
index d8b85fcd0d..fcbcf043fc 100644
--- a/src/calibre/gui2/library/models.py
+++ b/src/calibre/gui2/library/models.py
@@ -16,7 +16,7 @@ from calibre.gui2 import NONE, config, UNDEFINED_QDATE
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
@@ -928,11 +928,12 @@ class DeviceBooksModel(BooksModel): # {{{
         if index.isValid() and self.editable:
             cname = self.column_map[index.column()]
             if cname in ('title', 'authors') or \
-                    (cname == 'collections' and self.db.supports_collections()):
+                    (cname == 'collections' and \
+                     self.db.supports_collections() and \
+                     prefs['preserve_user_collections']):
                 flags |= Qt.ItemIsEditable
         return flags
 
-
     def search(self, text, reset=True):
         if not text or not text.strip():
             self.map = list(range(len(self.db)))
diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py
index 19daa1353c..09c1f8478b 100644
--- a/src/calibre/gui2/library/views.py
+++ b/src/calibre/gui2/library/views.py
@@ -15,7 +15,7 @@ from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
 from calibre.gui2.library import DEFAULT_SORT
 
@@ -500,7 +500,9 @@ class DeviceBooksView(BooksView): # {{{
         self.setAcceptDrops(False)
 
     def contextMenuEvent(self, event):
-        self.edit_collections_menu.setVisible(self._model.db.supports_collections())
+        self.edit_collections_menu.setVisible(
+                            self._model.db.supports_collections() and \
+                            prefs['preserve_user_collections'])
         self.context_menu.popup(event.globalPos())
         event.accept()
 
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 6452890883..590329ec13 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -473,6 +473,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
             self.search_restriction.setEnabled(False)
             for action in list(self.delete_menu.actions())[1:]:
                 action.setEnabled(False)
+            # Reset the view in case something changed while it was invisible
+            self.current_view().reset()
         self.set_number_of_books_shown()
 
 
diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py
index 69eee4d1ed..f24a6d2e30 100644
--- a/src/calibre/utils/config.py
+++ b/src/calibre/utils/config.py
@@ -698,6 +698,8 @@ def _prefs():
     # calibre server can execute searches
     c.add_opt('saved_searches', default={}, help=_('List of named saved searches'))
     c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
+    c.add_opt('preserve_user_collections', default=True,
+        help=_('Preserve all collections even if not in library metadata.'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c

From 169ec93134cce5b9a71a438478ed7953a2d8dbcc Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jun 2010 07:27:04 -0600
Subject: [PATCH 24/31] Fix multiple ratings displayed in Tag Browser for some
 legacy databases

---
 src/calibre/library/database2.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 7e4a879654..2983ac5e58 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -782,6 +782,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         icon=icon, tooltip=tooltip)
                                     for r in data if item_not_zero_func(r)]
 
+        # Needed for legacy databases that have multiple ratings that
+        # map to n stars
+        for r in categories['rating']:
+            for x in categories['rating']:
+                if r.name == x.name and r.id != x.id:
+                    r.count = r.count + x.count
+                    categories['rating'].remove(x)
+                    break
+
         # We delayed computing the standard formats category because it does not
         # use a view, but is computed dynamically
         categories['formats'] = []

From 41b4a5dd96c5f129a915445e9a715e2d1988fd3f Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jun 2010 08:30:18 -0600
Subject: [PATCH 25/31] Metadata download: Filter out non book results. Also
 sort results by availability of covers for the isbn. Fixes #5946 (fix file
 plugin postprocessing and update metadata download sorting)

---
 src/calibre/ebooks/metadata/fetch.py | 102 +++++++++++++++++++++++----
 1 file changed, 90 insertions(+), 12 deletions(-)

diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py
index d12c668e0d..db6ad0278d 100644
--- a/src/calibre/ebooks/metadata/fetch.py
+++ b/src/calibre/ebooks/metadata/fetch.py
@@ -3,17 +3,18 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, sys, textwrap, re
+import traceback, sys, textwrap, re, urllib2
 from threading import Thread
 
-from calibre import prints
+from calibre import prints, browser
 from calibre.utils.config import OptionParser
 from calibre.utils.logging import default_log
 from calibre.customize import Plugin
+from calibre.ebooks.metadata.library_thing import OPENLIBRARY
 
 metadata_config = None
 
-class MetadataSource(Plugin):
+class MetadataSource(Plugin): # {{{
 
     author = 'Kovid Goyal'
 
@@ -130,7 +131,9 @@ class MetadataSource(Plugin):
     def customization_help(self):
         return 'This plugin can only be customized using the GUI'
 
-class GoogleBooks(MetadataSource):
+    # }}}
+
+class GoogleBooks(MetadataSource): # {{{
 
     name = 'Google Books'
     description = _('Downloads metadata from Google Books')
@@ -145,8 +148,9 @@ class GoogleBooks(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
 
-class ISBNDB(MetadataSource):
+class ISBNDB(MetadataSource): # {{{
 
     name = 'IsbnDB'
     description = _('Downloads metadata from isbndb.com')
@@ -181,7 +185,9 @@ class ISBNDB(MetadataSource):
                 'and enter your access key below.')
         return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
 
-class Amazon(MetadataSource):
+    # }}}
+
+class Amazon(MetadataSource): # {{{
 
     name = 'Amazon'
     metadata_type = 'social'
@@ -198,7 +204,9 @@ class Amazon(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
-class LibraryThing(MetadataSource):
+    # }}}
+
+class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
@@ -207,7 +215,6 @@ class LibraryThing(MetadataSource):
     def fetch(self):
         if not self.isbn:
             return
-        from calibre import browser
         from calibre.ebooks.metadata import MetaInformation
         import json
         br = browser()
@@ -228,6 +235,7 @@ class LibraryThing(MetadataSource):
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
+    # }}}
 
 
 def result_index(source, result):
@@ -268,6 +276,27 @@ class MetadataSources(object):
         for s in self.sources:
             s.join()
 
+def filter_metadata_results(item):
+    keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
+    for keyword in keywords:
+        if item.publisher and keyword in item.publisher.lower():
+            return False
+    return True
+
+class HeadRequest(urllib2.Request):
+    def get_method(self):
+        return "HEAD"
+
+def check_for_covers(items):
+    opener = browser()
+    for item in items:
+       item.has_cover = False
+       try:
+            opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
+            item.has_cover = True
+       except:
+           pass # Cover not found
+
 def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):
     assert not(title is None and author is None and publisher is None and \
@@ -285,10 +314,59 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
     for fetcher in fetchers[1:]:
         merge_results(results, fetcher.results)
 
-    results = sorted(results, cmp=lambda x, y : cmp(
-            (x.comments.strip() if x.comments else ''),
-            (y.comments.strip() if y.comments else '')
-                                                  ), reverse=True)
+    results = list(filter(filter_metadata_results, results))
+
+    check_for_covers(results)
+
+    words = ("the", "a", "an", "of", "and")
+    prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
+    trailing_paren_pat = re.compile(r'\(.*\)$')
+    whitespace_pat = re.compile(r'\s+')
+
+    def sort_func(x, y):
+        def cleanup_title(s):
+            s = s.strip().lower()
+            s = prefix_pat.sub(' ', s)
+            s = trailing_paren_pat.sub('', s)
+            s = whitespace_pat.sub(' ', s)
+            return s.strip()
+
+        t = cleanup_title(title)
+        x_title = cleanup_title(x.title)
+        y_title = cleanup_title(y.title)
+
+        # prefer titles that start with the search title
+        tx = cmp(t, x_title)
+        ty = cmp(t, y_title)
+        result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
+
+        # then prefer titles that have a cover image
+        if result == 0:
+            result = -cmp(x.has_cover, y.has_cover)
+
+        # then prefer titles with the longest comment, with in 10%
+        if result == 0:
+            cx = len(x.comments.strip() if x.comments else '')
+            cy = len(y.comments.strip() if y.comments else '')
+            t = (cx + cy) / 20
+            result = cy - cx
+            if abs(result) < t:
+                result = 0
+
+        return result
+
+    results = sorted(results, cmp=sort_func)
+
+    # if for some reason there is no comment in the top selection, go looking for one
+    if len(results) > 1:
+        if not results[0].comments or len(results[0].comments) == 0:
+            for r in results[1:]:
+                if title.lower() == r.title[:len(title)].lower() and r.comments and len(r.comments):
+                    results[0].comments = r.comments
+                    break
+
+ #   for r in results:
+ #       print "{0:14.14} {1:30.30} {2:20.20} {3:6} {4}".format(r.isbn, r.title, r.publisher, len(r.comments if r.comments else ''), r.has_cover)
 
     return results, [(x.name, x.exception, x.tb) for x in fetchers]
 

From bb5ab06f3b9e7791b9793c64c6e486b950e3b441 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jun 2010 11:56:54 -0600
Subject: [PATCH 26/31] Fix #5951 (unable to retrieve news item)

---
 resources/recipes/national_post.recipe | 39 ++++++++------------------
 1 file changed, 11 insertions(+), 28 deletions(-)

diff --git a/resources/recipes/national_post.recipe b/resources/recipes/national_post.recipe
index 4fe188934c..00eb918d02 100644
--- a/resources/recipes/national_post.recipe
+++ b/resources/recipes/national_post.recipe
@@ -7,18 +7,18 @@ class NYTimes(BasicNewsRecipe):
     __author__  = 'Krittika Goyal'
     description = 'Canadian national newspaper'
     timefmt = ' [%d %b, %Y]'
-    needs_subscription = False
     language = 'en_CA'
+    needs_subscription = False
 
     no_stylesheets = True
     #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags_after  = dict(name='div', attrs={'class':'npStoryTools npWidth1-6 npRight npTxtStrong'})
     remove_tags = [
        dict(name='iframe'),
-       dict(name='div', attrs={'class':'story-tools'}),
+       dict(name='div', attrs={'class':['story-tools', 'npStoryTools npWidth1-6 npRight npTxtStrong']}),
        #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
        #dict(name='form', attrs={'onsubmit':''}),
-       #dict(name='table', attrs={'cellspacing':'0'}),
+       dict(name='ul', attrs={'class':'npTxtAlt npGroup npTxtCentre npStoryShare npTxtStrong npTxtDim'}),
     ]
 
    # def preprocess_html(self, soup):
@@ -37,7 +37,7 @@ class NYTimes(BasicNewsRecipe):
     def parse_index(self):
             soup = self.nejm_get_index()
 
-            div = soup.find(id='LegoText4')
+            div = soup.find(id='npContentMain')
 
             current_section = None
             current_articles = []
@@ -50,7 +50,7 @@ class NYTimes(BasicNewsRecipe):
                     current_section = self.tag_to_string(x)
                     current_articles = []
                     self.log('\tFound section:', current_section)
-                if current_section is not None and x.name == 'h3':
+                if current_section is not None and x.name == 'h5':
                     # Article found
                     title = self.tag_to_string(x)
                     a = x.find('a', href=lambda x: x and 'story' in x)
@@ -59,8 +59,8 @@ class NYTimes(BasicNewsRecipe):
                     url = a.get('href', False)
                     if not url or not title:
                         continue
-                    if url.startswith('story'):
-                         url = 'http://www.nationalpost.com/todays-paper/'+url
+                    #if url.startswith('story'):
+                    url = 'http://www.nationalpost.com/todays-paper/'+url
                     self.log('\t\tFound article:', title)
                     self.log('\t\t\t', url)
                     current_articles.append({'title': title, 'url':url,
@@ -70,28 +70,11 @@ class NYTimes(BasicNewsRecipe):
                 feeds.append((current_section, current_articles))
 
             return feeds
-
     def preprocess_html(self, soup):
-        story = soup.find(name='div', attrs={'class':'triline'})
-        page2_link = soup.find('p','pagenav')
-        if page2_link:
-            atag = page2_link.find('a',href=True)
-            if atag:
-                page2_url = atag['href']
-                if page2_url.startswith('story'):
-                         page2_url = 'http://www.nationalpost.com/todays-paper/'+page2_url
-                elif page2_url.startswith( '/todays-paper/story.html'):
-                    page2_url = 'http://www.nationalpost.com/'+page2_url
-                page2_soup = self.index_to_soup(page2_url)
-                if page2_soup:
-                    page2_content = page2_soup.find('div','story-content')
-                    if page2_content:
-                        full_story = BeautifulSoup('<div></div>')
-                        full_story.insert(0,story)
-                        full_story.insert(1,page2_content)
-                        story = full_story
+        story = soup.find(name='div', attrs={'id':'npContentMain'})
+        ##td = heading.findParent(name='td')
+        ##td.extract()
         soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
         body = soup.find(name='body')
         body.insert(0, story)
         return soup
-

From 985e65d3864fa6a8575c9bb0f76ea8089eab72fc Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jun 2010 12:06:34 -0600
Subject: [PATCH 27/31] Metadata download: Make cover check multithreaded

---
 src/calibre/ebooks/metadata/fetch.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py
index e7883d3757..0fd671f86a 100644
--- a/src/calibre/ebooks/metadata/fetch.py
+++ b/src/calibre/ebooks/metadata/fetch.py
@@ -287,15 +287,19 @@ class HeadRequest(urllib2.Request):
     def get_method(self):
         return "HEAD"
 
-def check_for_covers(items):
+def do_cover_check(item):
     opener = browser()
-    for item in items:
-       item.has_cover = False
-       try:
-            opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
-            item.has_cover = True
-       except:
-           pass # Cover not found
+    item.has_cover = False
+    try:
+        opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
+        item.has_cover = True
+    except:
+        pass # Cover not found
+
+def check_for_covers(items):
+    threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
+    for t in threads: t.start()
+    for t in threads: t.join()
 
 def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):

From e7eb5b69657de4d051bd1900a27f16f501afb5b3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 24 Jun 2010 12:07:33 -0600
Subject: [PATCH 28/31] Fix #5937 ("New Scientist" recipe problems)

---
 resources/recipes/new_scientist.recipe | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/resources/recipes/new_scientist.recipe b/resources/recipes/new_scientist.recipe
index 1727a926ed..b40be458bc 100644
--- a/resources/recipes/new_scientist.recipe
+++ b/resources/recipes/new_scientist.recipe
@@ -32,15 +32,16 @@ class NewScientist(BasicNewsRecipe):
                         }
     preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','nsblgposts','hldgalcols']})]
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
-                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools']})
+                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
                     ,dict(name='meta' , attrs={'name' :'description'                       })
+                    ,dict(name='a'    , attrs={'rel'  :'tag'                                })
                   ]
-    remove_tags_after = dict(attrs={'class':'nbpcopy'})
+    remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
     remove_attributes = ['height','width']
 
     feeds          = [

From 716b8a0905ce02205a35a16b5e03a46742f9660d Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 19:40:28 +0100
Subject: [PATCH 29/31] Changes to FAQ

---
 src/calibre/manual/faq.rst | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst
index 4c89ed48c0..351eb6a812 100644
--- a/src/calibre/manual/faq.rst
+++ b/src/calibre/manual/faq.rst
@@ -107,17 +107,27 @@ How does |app| manage collections on my SONY reader?
 When |app| connects with the device, it retrieves all collections for the books on the device. The collections
 of which books are members are shown on the device view.
 
-When you send a book to the device, |app| will if necessary create new collections based on the metadata for
-that book, then add the book to the collections. By default, collections are created from tags and series. You
-can control what metadata is used by going to Preferences->Plugins->Device Interface plugins and customizing
-the SONY device interface plugin.
+When you send a book to the device, |app| will add the book to collections based on the metadata for that book. By
+default, collections are created from tags and series. You can control what metadata is used by going to
+Preferences->Plugins->Device Interface plugins and customizing the SONY device interface plugin. If you remove all
+values, |app| will not add the book to any collection.
 
-|app| will not delete already existing collections for a book on your device when you resend the book to the
-device. To ensure that the collections are based only on current |app| metadata, first delete the books from
-the device, and then resend the books.
+Collection management is largely controlled by 'Preserve device collections' found at Preferences->Add/Save->Sending
+to device. If checked (the default), managing collections is left to the user; |app| will not delete already
+existing collections for a book on your device when you resend the book to the device, but |app| will add the book to
+collections if necessary.  To ensure that the collections for a book are based only on current |app| metadata, first
+delete the books from the device, then resend the books.  You can edit collections directly on the device view by
+double-clicking or right-clicking in the collections column.
 
-You can edit collections on the device in the device view in |app| by double clicking or right clicking on the
-collections field. This is the only way to remove a book from a collection.
+If 'Preserve device collections' is not checked, then |app| will manage collections. Collections will be built using
+|app| metadata exclusively.  Sending a book to the device will correct the collections for that book so its
+collections exactly match the book's metadata. Collections are added and deleted as necessary.  Editing collections on
+the device pane is not permitted, because collections not in the metadata will be removed automatically.
+
+In summary, check 'Preserve device collections' if you want to manage collections yourself.  Collections for a book
+will never be removed by |app|, but can be removed by you by editing on the device view.  Uncheck 'Preserve device
+collections' if you want |app| to manage the collections, adding books to and removing books from collections as
+needed.  
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From 91cd75909e1d1eae0f2c324267dba2e6afd36d58 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 21:26:04 +0100
Subject: [PATCH 30/31] Fix renaming bug where multiple items were renamed to
 the same thing

---
 src/calibre/gui2/actions.py                 | 5 +++--
 src/calibre/gui2/dialogs/tag_list_editor.py | 5 ++++-
 src/calibre/gui2/tag_view.py                | 4 +++-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/actions.py b/src/calibre/gui2/actions.py
index f3f6ee604c..43a657ae67 100644
--- a/src/calibre/gui2/actions.py
+++ b/src/calibre/gui2/actions.py
@@ -839,10 +839,11 @@ class EditMetadataAction(object): # {{{
         d = TagListEditor(self, tag_to_match=None, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
-            to_rename = d.to_rename # dict of new text to old id
+            to_rename = d.to_rename # dict of new text to old ids
             to_delete = d.to_delete # list of ids
             for text in to_rename:
-                model.rename_collection(old_id=to_rename[text], new_name=unicode(text))
+                for old_id in to_rename[text]:
+                    model.rename_collection(old_id, new_name=unicode(text))
             for item in to_delete:
                 model.delete_collection_using_id(item)
             self.upload_collections(model.db, view=view, oncard=oncard)
diff --git a/src/calibre/gui2/dialogs/tag_list_editor.py b/src/calibre/gui2/dialogs/tag_list_editor.py
index 6d91ed2ee2..9eb368e5e4 100644
--- a/src/calibre/gui2/dialogs/tag_list_editor.py
+++ b/src/calibre/gui2/dialogs/tag_list_editor.py
@@ -73,7 +73,10 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 return
         if item.text() != self.item_before_editing.text():
             (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            self.to_rename[item.text()] = id
+            if item.text() not in self.to_rename:
+                self.to_rename[item.text()] = [id]
+            else:
+                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 140b1e1e52..34b9bd1426 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -720,7 +720,9 @@ class TagBrowserMixin(object): # {{{
                 delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
             if rename_func:
                 for text in to_rename:
-                    rename_func(old_id=to_rename[text], new_name=unicode(text))
+                        for old_id in to_rename[text]:
+                            print 'rename', old_id, text
+                            rename_func(old_id, new_name=unicode(text))
                 for item in to_delete:
                     delete_func(item)
 

From f299bf01ad9a643e37a74a969302c96f3e9c0f1c Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Thu, 24 Jun 2010 21:37:23 +0100
Subject: [PATCH 31/31] Remove print statement

---
 src/calibre/gui2/tag_view.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py
index 34b9bd1426..189caea6ea 100644
--- a/src/calibre/gui2/tag_view.py
+++ b/src/calibre/gui2/tag_view.py
@@ -721,7 +721,6 @@ class TagBrowserMixin(object): # {{{
             if rename_func:
                 for text in to_rename:
                         for old_id in to_rename[text]:
-                            print 'rename', old_id, text
                             rename_func(old_id, new_name=unicode(text))
                 for item in to_delete:
                     delete_func(item)