KG updates

resources/recipes/lrb.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 lrb.co.uk
 '''
@@ -8,32 +8,38 @@ lrb.co.uk
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class LondonReviewOfBooks(BasicNewsRecipe):
-    title                 = u'London Review of Books'
-    __author__            = u'Darko Miletic'
-    description           = u'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
-    category              = 'news, literature, England'
-    publisher             = 'London Review of Books'
-    oldest_article        = 7
+    title                 = 'London Review of Books (free)'
+    __author__            = 'Darko Miletic'
+    description           = 'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB ltd.'
+    oldest_article        = 15
     max_articles_per_feed = 100
     language              = 'en_GB'
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
+    publication_type      = 'magazine'
+    masthead_url          = 'http://www.lrb.co.uk/assets/images/lrb_logo_big.gif'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
 
-    conversion_options = {  
+    conversion_options = {
                              'comments'  : description
                             ,'tags'      : category
                             ,'language'  : language
                             ,'publisher' : publisher
                          }
-    
-    keep_only_tags = [dict(name='div' , attrs={'id'   :'main'})]
-    remove_tags = [
-                    dict(name='div' , attrs={'class':['pagetools','issue-nav-controls','nocss']})
-                   ,dict(name='div' , attrs={'id'   :['mainmenu','precontent','otherarticles']     })
-                   ,dict(name='span', attrs={'class':['inlineright','article-icons']})
-                   ,dict(name='ul'  , attrs={'class':'article-controls'})
-                   ,dict(name='p'   , attrs={'class':'meta-info'       })
-                  ]
+
+    keep_only_tags = [dict(attrs={'class':['article-body indent','letters','article-list']})]
+    remove_attributes = ['width','height']
 
     feeds = [(u'London Review of Books', u'http://www.lrb.co.uk/lrbrss.xml')]
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup('http://www.lrb.co.uk/')
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        if cover_item:
+           cover_url = 'http://www.lrb.co.uk' + cover_item.a.img['src']
+        return cover_url
+

resources/recipes/lrb_payed.recipe

@@ -0,0 +1,75 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+lrb.co.uk
+'''
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonReviewOfBooksPayed(BasicNewsRecipe):
+    title                 = 'London Review of Books'
+    __author__            = 'Darko Miletic'
+    description           = 'Subscription content. Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB Ltd.'
+    max_articles_per_feed = 100
+    language              = 'en_GB'
+    no_stylesheets        = True
+    delay                 = 1
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    INDEX                 = 'http://www.lrb.co.uk'
+    LOGIN                 = INDEX + '/login'
+    masthead_url          = INDEX + '/assets/images/lrb_logo_big.gif'
+    needs_subscription    = True
+    publication_type      = 'magazine'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.LOGIN)
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        lrbtitle = self.title
+        if  cover_item:
+            self.cover_url = self.INDEX + cover_item.a.img['src']
+            content = self.INDEX + cover_item.a['href']
+            soup2 = self.index_to_soup(content)
+            sitem = soup2.find(attrs={'class':'article-list'})
+            lrbtitle = soup2.head.title.string
+            for item in sitem.findAll('a',attrs={'class':'title'}):
+                description = u''
+                title_prefix = u''
+                feed_link = item
+                if feed_link.has_key('href'):
+                    url   = self.INDEX + feed_link['href']
+                    title = title_prefix + self.tag_to_string(feed_link)
+                    date  = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+        return [(lrbtitle, articles)]
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    keep_only_tags = [dict(name='div' , attrs={'class':['article-body indent','letters']})]
+    remove_attributes = ['width','height']

resources/recipes/national_post.recipe

@@ -7,18 +7,18 @@ class NYTimes(BasicNewsRecipe):
     __author__  = 'Krittika Goyal'
     description = 'Canadian national newspaper'
     timefmt = ' [%d %b, %Y]'
-    needs_subscription = False
     language = 'en_CA'
+    needs_subscription = False
 
     no_stylesheets = True
     #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags_after  = dict(name='div', attrs={'class':'npStoryTools npWidth1-6 npRight npTxtStrong'})
     remove_tags = [
        dict(name='iframe'),
-       dict(name='div', attrs={'class':'story-tools'}),
+       dict(name='div', attrs={'class':['story-tools', 'npStoryTools npWidth1-6 npRight npTxtStrong']}),
        #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
        #dict(name='form', attrs={'onsubmit':''}),
-       #dict(name='table', attrs={'cellspacing':'0'}),
+       dict(name='ul', attrs={'class':'npTxtAlt npGroup npTxtCentre npStoryShare npTxtStrong npTxtDim'}),
     ]
 
    # def preprocess_html(self, soup):
@@ -37,7 +37,7 @@ class NYTimes(BasicNewsRecipe):
     def parse_index(self):
             soup = self.nejm_get_index()
 
-            div = soup.find(id='LegoText4')
+            div = soup.find(id='npContentMain')
 
             current_section = None
             current_articles = []
@@ -50,7 +50,7 @@ class NYTimes(BasicNewsRecipe):
                     current_section = self.tag_to_string(x)
                     current_articles = []
                     self.log('\tFound section:', current_section)
-                if current_section is not None and x.name == 'h3':
+                if current_section is not None and x.name == 'h5':
                     # Article found
                     title = self.tag_to_string(x)
                     a = x.find('a', href=lambda x: x and 'story' in x)
@@ -59,8 +59,8 @@ class NYTimes(BasicNewsRecipe):
                     url = a.get('href', False)
                     if not url or not title:
                         continue
-                    if url.startswith('story'):
-                         url = 'http://www.nationalpost.com/todays-paper/'+url
+                    #if url.startswith('story'):
+                    url = 'http://www.nationalpost.com/todays-paper/'+url
                     self.log('\t\tFound article:', title)
                     self.log('\t\t\t', url)
                     current_articles.append({'title': title, 'url':url,
@@ -70,28 +70,11 @@ class NYTimes(BasicNewsRecipe):
                 feeds.append((current_section, current_articles))
 
             return feeds
-
     def preprocess_html(self, soup):
-        story = soup.find(name='div', attrs={'class':'triline'})
-        page2_link = soup.find('p','pagenav')
-        if page2_link:
-            atag = page2_link.find('a',href=True)
-            if atag:
-                page2_url = atag['href']
-                if page2_url.startswith('story'):
-                         page2_url = 'http://www.nationalpost.com/todays-paper/'+page2_url
-                elif page2_url.startswith( '/todays-paper/story.html'):
-                    page2_url = 'http://www.nationalpost.com/'+page2_url
-                page2_soup = self.index_to_soup(page2_url)
-                if page2_soup:
-                    page2_content = page2_soup.find('div','story-content')
-                    if page2_content:
-                        full_story = BeautifulSoup('<div></div>')
-                        full_story.insert(0,story)
-                        full_story.insert(1,page2_content)
-                        story = full_story
+        story = soup.find(name='div', attrs={'id':'npContentMain'})
+        ##td = heading.findParent(name='td')
+        ##td.extract()
         soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
         body = soup.find(name='body')
         body.insert(0, story)
         return soup
-

resources/recipes/new_scientist.recipe

@@ -32,15 +32,16 @@ class NewScientist(BasicNewsRecipe):
                         }
     preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','nsblgposts','hldgalcols']})]
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
-                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools']})
+                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
                     ,dict(name='meta' , attrs={'name' :'description'                       })
+                    ,dict(name='a'    , attrs={'rel'  :'tag'                                })
                   ]
-    remove_tags_after = dict(attrs={'class':'nbpcopy'})
+    remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
     remove_attributes = ['height','width']
 
     feeds          = [

src/calibre/customize/ui.py

@@ -151,13 +151,13 @@ def reread_filetype_plugins():
 
 
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
-    occasion = {'import':_on_import, 'preprocess':_on_preprocess,
+    occasion_plugins = {'import':_on_import, 'preprocess':_on_preprocess,
                 'postprocess':_on_postprocess}[occasion]
     customization = config['plugin_customization']
     if ft is None:
         ft = os.path.splitext(path_to_file)[-1].lower().replace('.', '')
     nfp = path_to_file
-    for plugin in occasion.get(ft, []):
+    for plugin in occasion_plugins.get(ft, []):
         if is_disabled(plugin):
             continue
         plugin.site_customization = customization.get(plugin.name, '')

src/calibre/devices/prs505/driver.py

@@ -99,7 +99,7 @@ class PRS505(USBMS):
         if self._card_b_prefix is not None:
             if not write_cache(self._card_b_prefix):
                 self._card_b_prefix = None
-
+        self.booklist_class.rebuild_collections = self.rebuild_collections
 
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
@@ -145,7 +145,7 @@ class PRS505(USBMS):
                 blists[i] = booklists[i]
         opts = self.settings()
         if opts.extra_customization:
-            collections = [x.strip() for x in
+            collections = [x.lower().strip() for x in
                     opts.extra_customization.split(',')]
         else:
             collections = []
@@ -156,4 +156,10 @@ class PRS505(USBMS):
         USBMS.sync_booklists(self, booklists, end_session=end_session)
         debug_print('PRS505: finished sync_booklists')
 
+    def rebuild_collections(self, booklist, oncard):
+        debug_print('PRS505: started rebuild_collections on card', oncard)
+        c = self.initialize_XML_cache()
+        c.rebuild_collections(booklist, {'carda':1, 'cardb':2}.get(oncard, 0))
+        c.write()
+        debug_print('PRS505: finished rebuild_collections')
 

src/calibre/devices/prs505/sony_cache.py

@@ -6,10 +6,8 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, time
-from pprint import pprint
 from base64 import b64decode
 from uuid import uuid4
-
 from lxml import etree
 
 from calibre import prints, guess_type
@@ -62,8 +60,7 @@ class XMLCache(object):
 
     def __init__(self, paths, prefixes, use_author_sort):
         if DEBUG:
-            debug_print('Building XMLCache...')
-            pprint(paths)
+            debug_print('Building XMLCache...', paths)
         self.paths = paths
         self.prefixes = prefixes
         self.use_author_sort = use_author_sort
@@ -151,15 +148,14 @@ class XMLCache(object):
                 else:
                     seen.add(title)
 
-    def get_playlist_map(self):
-        debug_print('Start get_playlist_map')
+    def build_playlist_id_map(self):
+        debug_print('Start build_playlist_id_map')
         ans = {}
         self.ensure_unique_playlist_titles()
         debug_print('after ensure_unique_playlist_titles')
         self.prune_empty_playlists()
-        debug_print('get_playlist_map loop')
         for i, root in self.record_roots.items():
-            debug_print('get_playlist_map loop', i)
+            debug_print('build_playlist_id_map loop', i)
             id_map = self.build_id_map(root)
             ans[i] = []
             for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -170,9 +166,23 @@ class XMLCache(object):
                     if record is not None:
                         items.append(record)
                 ans[i].append((playlist.get('title'), items))
-        debug_print('end get_playlist_map')
+        debug_print('end build_playlist_id_map')
         return ans
 
+    def build_id_playlist_map(self, bl_index):
+        debug_print('Start build_id_playlist_map')
+        pmap = self.build_playlist_id_map()[bl_index]
+        playlist_map = {}
+        for title, records in pmap:
+            for record in records:
+                path = record.get('path', None)
+                if path:
+                    if path not in playlist_map:
+                        playlist_map[path] = []
+                    playlist_map[path].append(title)
+        debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
+        return playlist_map
+
     def get_or_create_playlist(self, bl_idx, title):
         root = self.record_roots[bl_idx]
         for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -192,8 +202,7 @@ class XMLCache(object):
     # }}}
 
     def fix_ids(self): # {{{
-        if DEBUG:
-            debug_print('Running fix_ids()')
+        debug_print('Running fix_ids()')
 
         def ensure_numeric_ids(root):
             idmap = {}
@@ -276,38 +285,19 @@ class XMLCache(object):
     def update_booklist(self, bl, bl_index):
         if bl_index not in self.record_roots:
             return
-        if DEBUG:
-            debug_print('Updating JSON cache:', bl_index)
+        debug_print('Updating JSON cache:', bl_index)
+        playlist_map = self.build_id_playlist_map(bl_index)
         root = self.record_roots[bl_index]
-        pmap = self.get_playlist_map()[bl_index]
-        playlist_map = {}
-        for title, records in pmap:
-            for record in records:
-                path = record.get('path', None)
-                if path:
-                    if path not in playlist_map:
-                        playlist_map[path] = []
-                    playlist_map[path].append(title)
-
         lpath_map = self.build_lpath_map(root)
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
                 title = record.get('title', None)
                 if title is not None and title != book.title:
-                    if DEBUG:
-                        debug_print('Renaming title', book.title, 'to', title)
+                    debug_print('Renaming title', book.title, 'to', title)
                     book.title = title
-# We shouldn't do this for Sonys, because the reader strips
-# all but the first author.
-#                authors = record.get('author', None)
-#                if authors is not None:
-#                    authors = string_to_authors(authors)
-#                    if authors != book.authors:
-#                        if DEBUG:
-#                            prints('Renaming authors', book.authors, 'to',
-#                                    authors)
-#                        book.authors = authors
+                    # Don't set the author, because the reader strips all but
+                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -318,45 +308,50 @@ class XMLCache(object):
                             book.thumbnail = raw
                             break
                     break
-                if book.lpath in playlist_map:
-                    tags = playlist_map[book.lpath]
-                    book.device_collections = tags
+                book.device_collections = playlist_map.get(book.lpath, [])
         debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
-        debug_print('Starting update XML from JSON')
-        playlist_map = self.get_playlist_map()
-
+        debug_print('Starting update', collections_attributes)
         for i, booklist in booklists.items():
-            if DEBUG:
-                debug_print('Updating XML Cache:', i)
+            playlist_map = self.build_id_playlist_map(i)
+            debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
             lpath_map = self.build_lpath_map(root)
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
-#                record = self.book_by_lpath(book.lpath, root)
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
                 self.update_text_record(record, book, path, i)
+                # Ensure the collections in the XML database are recorded for
+                # this book
+                if book.device_collections is None:
+                    book.device_collections = []
+                book.device_collections = playlist_map.get(book.lpath, [])
+            self.update_playlists(i, root, booklist, collections_attributes)
+        # Update the device collections because update playlist could have added
+        # some new ones.
+        debug_print('In update/ Starting refresh of device_collections')
+        for i, booklist in booklists.items():
+            playlist_map = self.build_id_playlist_map(i)
+            for book in booklist:
+                book.device_collections = playlist_map.get(book.lpath, [])
+        self.fix_ids()
+        debug_print('Finished update')
 
-            bl_pmap = playlist_map[i]
-            self.update_playlists(i, root, booklist, bl_pmap,
-                    collections_attributes)
-
+    def rebuild_collections(self, booklist, bl_index):
+        if bl_index not in self.record_roots:
+            return
+        root = self.record_roots[bl_index]
+        self.update_playlists(bl_index, root, booklist, [])
         self.fix_ids()
 
-        # This is needed to update device_collections
-        for i, booklist in booklists.items():
-            self.update_booklist(booklist, i)
-        debug_print('Finished update XML from JSON')
-
-    def update_playlists(self, bl_index, root, booklist, playlist_map,
-            collections_attributes):
-        debug_print('Starting update_playlists')
+    def update_playlists(self, bl_index, root, booklist, collections_attributes):
+        debug_print('Starting update_playlists', collections_attributes, bl_index)
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
@@ -372,10 +367,8 @@ class XMLCache(object):
                     rec.set('id', str(self.max_id(root)+1))
             ids = [x.get('id', None) for x in records]
             if None in ids:
-                if DEBUG:
-                    debug_print('WARNING: Some <text> elements do not have ids')
-                    ids = [x for x in ids if x is not None]
-
+                debug_print('WARNING: Some <text> elements do not have ids')
+                ids = [x for x in ids if x is not None]
             playlist = self.get_or_create_playlist(bl_index, category)
             playlist_ids = []
             for item in playlist:
@@ -544,10 +537,5 @@ class XMLCache(object):
                         break
                 self.namespaces[i] = ns
 
-#        if DEBUG:
-#            debug_print('Found nsmaps:')
-#            pprint(self.nsmaps)
-#            debug_print('Found namespaces:')
-#            pprint(self.namespaces)
     # }}}
 

src/calibre/devices/usbms/books.py

@@ -11,6 +11,7 @@ from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
+from calibre.utils.config import prefs
 
 class Book(MetaInformation):
 
@@ -76,7 +77,7 @@ class Book(MetaInformation):
         in C{other} takes precedence, unless the information in C{other} is NULL.
         '''
 
-        MetaInformation.smart_update(self, other)
+        MetaInformation.smart_update(self, other, replace_tags=True)
 
         for attr in self.BOOK_ATTRS:
             if hasattr(other, attr):
@@ -132,7 +133,9 @@ class CollectionsBookList(BookList):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
-        collection_attributes = list(collection_attributes)+['device_collections']
+        collection_attributes = list(collection_attributes)
+        if prefs['preserve_user_collections']:
+            collection_attributes += ['device_collections']
         for attr in collection_attributes:
             attr = attr.strip()
             for book in self:
@@ -167,3 +170,15 @@ class CollectionsBookList(BookList):
                 books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
         return collections
 
+    def rebuild_collections(self, booklist, oncard):
+        '''
+        For each book in the booklist for the card oncard, remove it from all
+        its current collections, then add it to the collections specified in
+        device_collections.
+
+        oncard is None for the main memory, carda for card A, cardb for card B,
+        etc.
+
+        booklist is the object created by the :method:`books` call above.
+        '''
+        pass

src/calibre/ebooks/conversion/preprocess.py

@@ -107,9 +107,21 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
 
-    def __call__(self, data):
+    def __call__(self, data, add_namespace=False):
+        from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
-        return data
+        if not add_namespace:
+            return data
+        ans, namespaced = [], False
+        for line in data.splitlines():
+            ll = line.lstrip()
+            if not (namespaced or ll.startswith('@import') or
+                        ll.startswith('@charset')):
+                ans.append(XHTML_CSS_NAMESPACE.strip())
+                namespaced = True
+            ans.append(line)
+
+        return u'\n'.join(ans)
 
 class HTMLPreProcessor(object):
 

src/calibre/ebooks/html/input.py

@@ -20,7 +20,7 @@ from itertools import izip
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
-from calibre.constants import islinux, isfreebsd
+from calibre.constants import islinux, isfreebsd, iswindows
 from calibre import unicode_path
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
@@ -32,9 +32,14 @@ class Link(object):
 
     @classmethod
     def url_to_local_path(cls, url, base):
-        path = urlunparse(('', '', url.path, url.params, url.query, ''))
+        path = url.path
+        isabs = False
+        if iswindows and path.startswith('/'):
+            path = path[1:]
+            isabs = True
+        path = urlunparse(('', '', path, url.params, url.query, ''))
         path = unquote(path)
-        if os.path.isabs(path):
+        if isabs or os.path.isabs(path):
             return path
         return os.path.abspath(os.path.join(base, path))
 
@@ -307,6 +312,7 @@ class HTMLInput(InputFormatPlugin):
             xpath
         from calibre import guess_type
         import cssutils
+        self.OEB_STYLES = OEB_STYLES
         oeb = create_oebbook(log, None, opts, self,
                 encoding=opts.input_encoding, populate=False)
         self.oeb = oeb
@@ -371,7 +377,7 @@ class HTMLInput(InputFormatPlugin):
             rewrite_links(item.data, partial(self.resource_adder, base=dpath))
 
         for item in oeb.manifest.values():
-            if item.media_type in OEB_STYLES:
+            if item.media_type in self.OEB_STYLES:
                 dpath = None
                 for path, href in self.added_resources.items():
                     if href == item.href:
@@ -409,12 +415,30 @@ class HTMLInput(InputFormatPlugin):
         oeb.container = DirContainer(os.getcwdu(), oeb.log)
         return oeb
 
+    def link_to_local_path(self, link_, base=None):
+        if not isinstance(link_, unicode):
+            try:
+                link_ = link_.decode('utf-8', 'error')
+            except:
+                self.log.warn('Failed to decode link %r. Ignoring'%link_)
+                return None, None
+        try:
+            l = Link(link_, base if base else os.getcwdu())
+        except:
+            self.log.exception('Failed to process link: %r'%link_)
+            return None, None
+        if l.path is None:
+            # Not a local resource
+            return None, None
+        link = l.path.replace('/', os.sep).strip()
+        frag = l.fragment
+        if not link:
+            return None, None
+        return link, frag
 
     def resource_adder(self, link_, base=None):
-        link = self.urlnormalize(link_)
-        link, frag = self.urldefrag(link)
-        link = unquote(link).replace('/', os.sep)
-        if not link.strip():
+        link, frag = self.link_to_local_path(link_, base=base)
+        if link is None:
             return link_
         try:
             if base and not os.path.isabs(link):
@@ -442,6 +466,9 @@ class HTMLInput(InputFormatPlugin):
 
             item = self.oeb.manifest.add(id, href, media_type)
             item.html_input_href = bhref
+            if guessed in self.OEB_STYLES:
+                item.override_css_fetch = partial(
+                        self.css_import_handler, os.path.dirname(link))
             item.data
             self.added_resources[link] = href
 
@@ -450,7 +477,17 @@ class HTMLInput(InputFormatPlugin):
             nlink = '#'.join((nlink, frag))
         return nlink
 
-
+    def css_import_handler(self, base, href):
+        link, frag = self.link_to_local_path(href, base=base)
+        if link is None or not os.access(link, os.R_OK) or os.path.isdir(link):
+            return (None, None)
+        try:
+            raw = open(link, 'rb').read().decode('utf-8', 'replace')
+            raw = self.oeb.css_preprocessor(raw, add_namespace=True)
+        except:
+            self.log.exception('Failed to read CSS file: %r'%link)
+            return (None, None)
+        return (None, raw)
 
 
 

src/calibre/ebooks/metadata/__init__.py

@@ -268,7 +268,7 @@ class MetaInformation(object):
                   ):
             prints(x, getattr(self, x, 'None'))
 
-    def smart_update(self, mi):
+    def smart_update(self, mi, replace_tags=False):
         '''
         Merge the information in C{mi} into self. In case of conflicts, the information
         in C{mi} takes precedence, unless the information in mi is NULL.
@@ -291,7 +291,10 @@ class MetaInformation(object):
                     setattr(self, attr, val)
 
         if mi.tags:
-            self.tags += mi.tags
+            if replace_tags:
+                self.tags = mi.tags
+            else:
+                self.tags += mi.tags
         self.tags = list(set(self.tags))
 
         if mi.author_sort_map:

src/calibre/ebooks/metadata/fetch.py

@@ -3,17 +3,18 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, sys, textwrap, re
+import traceback, sys, textwrap, re, urllib2
 from threading import Thread
 
-from calibre import prints
+from calibre import prints, browser
 from calibre.utils.config import OptionParser
 from calibre.utils.logging import default_log
 from calibre.customize import Plugin
+from calibre.ebooks.metadata.library_thing import OPENLIBRARY
 
 metadata_config = None
 
-class MetadataSource(Plugin):
+class MetadataSource(Plugin): # {{{
 
     author = 'Kovid Goyal'
 
@@ -130,7 +131,9 @@ class MetadataSource(Plugin):
     def customization_help(self):
         return 'This plugin can only be customized using the GUI'
 
-class GoogleBooks(MetadataSource):
+    # }}}
+
+class GoogleBooks(MetadataSource): # {{{
 
     name = 'Google Books'
     description = _('Downloads metadata from Google Books')
@@ -145,8 +148,9 @@ class GoogleBooks(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
 
-class ISBNDB(MetadataSource):
+class ISBNDB(MetadataSource): # {{{
 
     name = 'IsbnDB'
     description = _('Downloads metadata from isbndb.com')
@@ -181,7 +185,9 @@ class ISBNDB(MetadataSource):
                 'and enter your access key below.')
         return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
 
-class Amazon(MetadataSource):
+    # }}}
+
+class Amazon(MetadataSource): # {{{
 
     name = 'Amazon'
     metadata_type = 'social'
@@ -198,7 +204,9 @@ class Amazon(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
-class LibraryThing(MetadataSource):
+    # }}}
+
+class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
@@ -207,7 +215,6 @@ class LibraryThing(MetadataSource):
     def fetch(self):
         if not self.isbn:
             return
-        from calibre import browser
         from calibre.ebooks.metadata import MetaInformation
         import json
         br = browser()
@@ -228,6 +235,7 @@ class LibraryThing(MetadataSource):
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
+    # }}}
 
 
 def result_index(source, result):
@@ -268,6 +276,31 @@ class MetadataSources(object):
         for s in self.sources:
             s.join()
 
+def filter_metadata_results(item):
+    keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
+    for keyword in keywords:
+        if item.publisher and keyword in item.publisher.lower():
+            return False
+    return True
+
+class HeadRequest(urllib2.Request):
+    def get_method(self):
+        return "HEAD"
+
+def do_cover_check(item):
+    opener = browser()
+    item.has_cover = False
+    try:
+        opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
+        item.has_cover = True
+    except:
+        pass # Cover not found
+
+def check_for_covers(items):
+    threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
+    for t in threads: t.start()
+    for t in threads: t.join()
+
 def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):
     assert not(title is None and author is None and publisher is None and \
@@ -285,10 +318,60 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
     for fetcher in fetchers[1:]:
         merge_results(results, fetcher.results)
 
-    results = sorted(results, cmp=lambda x, y : cmp(
-            (x.comments.strip() if x.comments else ''),
-            (y.comments.strip() if y.comments else '')
-                                                  ), reverse=True)
+    results = list(filter(filter_metadata_results, results))
+
+    check_for_covers(results)
+
+    words = ("the", "a", "an", "of", "and")
+    prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
+    trailing_paren_pat = re.compile(r'\(.*\)$')
+    whitespace_pat = re.compile(r'\s+')
+
+    def sort_func(x, y):
+
+        def cleanup_title(s):
+            s = s.strip().lower()
+            s = prefix_pat.sub(' ', s)
+            s = trailing_paren_pat.sub('', s)
+            s = whitespace_pat.sub(' ', s)
+            return s.strip()
+
+        t = cleanup_title(title)
+        x_title = cleanup_title(x.title)
+        y_title = cleanup_title(y.title)
+
+        # prefer titles that start with the search title
+        tx = cmp(t, x_title)
+        ty = cmp(t, y_title)
+        result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
+
+        # then prefer titles that have a cover image
+        if result == 0:
+            result = -cmp(x.has_cover, y.has_cover)
+
+        # then prefer titles with the longest comment, with in 10%
+        if result == 0:
+            cx = len(x.comments.strip() if x.comments else '')
+            cy = len(y.comments.strip() if y.comments else '')
+            t = (cx + cy) / 20
+            result = cy - cx
+            if abs(result) < t:
+                result = 0
+
+        return result
+
+    results = sorted(results, cmp=sort_func)
+
+    # if for some reason there is no comment in the top selection, go looking for one
+    if len(results) > 1:
+        if not results[0].comments or len(results[0].comments) == 0:
+            for r in results[1:]:
+                if title.lower() == r.title[:len(title)].lower() and r.comments and len(r.comments):
+                    results[0].comments = r.comments
+                    break
+
+ #   for r in results:
+ #       print "{0:14.14} {1:30.30} {2:20.20} {3:6} {4}".format(r.isbn, r.title, r.publisher, len(r.comments if r.comments else ''), r.has_cover)
 
     return results, [(x.name, x.exception, x.tb) for x in fetchers]
 

src/calibre/ebooks/oeb/base.py

@@ -17,6 +17,7 @@ from urlparse import urljoin
 
 from lxml import etree, html
 from cssutils import CSSParser
+from cssutils.css import CSSRule
 
 import calibre
 from calibre.constants import filesystem_encoding
@@ -762,6 +763,7 @@ class Manifest(object):
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
+            self.override_css_fetch = None
             self.spine_position = None
             self.linear = True
             if loader is None and data is None:
@@ -982,15 +984,40 @@ class Manifest(object):
 
 
         def _parse_css(self, data):
+
+            def get_style_rules_from_import(import_rule):
+                ans = []
+                if not import_rule.styleSheet:
+                    return ans
+                rules = import_rule.styleSheet.cssRules
+                for rule in rules:
+                    if rule.type == CSSRule.IMPORT_RULE:
+                        ans.extend(get_style_rules_from_import(rule))
+                    elif rule.type in (CSSRule.FONT_FACE_RULE,
+                            CSSRule.STYLE_RULE):
+                        ans.append(rule)
+                return ans
+
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
-            data = self.oeb.css_preprocessor(data)
-            data = XHTML_CSS_NAMESPACE + data
+            data = self.oeb.css_preprocessor(data, add_namespace=True)
             parser = CSSParser(loglevel=logging.WARNING,
-                               fetcher=self._fetch_css,
+                               fetcher=self.override_css_fetch or self._fetch_css,
                                log=_css_logger)
             data = parser.parseString(data, href=self.href)
             data.namespaces['h'] = XHTML_NS
+            import_rules = list(data.cssRules.rulesOfType(CSSRule.IMPORT_RULE))
+            rules_to_append = []
+            insert_index = None
+            for r in data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
+                insert_index = data.cssRules.index(r)
+                break
+            for rule in import_rules:
+                rules_to_append.extend(get_style_rules_from_import(rule))
+            for r in reversed(rules_to_append):
+                data.insertRule(r, index=insert_index)
+            for rule in import_rules:
+                data.deleteRule(rule)
             return data
 
         def _fetch_css(self, path):

src/calibre/ebooks/oeb/iterator.py

@@ -139,11 +139,18 @@ class EbookIterator(object):
                     if id != -1:
                         families = [unicode(f) for f in QFontDatabase.applicationFontFamilies(id)]
                         if family:
-                            family = family.group(1).strip().replace('"', '')
-                            bad_map[family] = families[0]
-                            if family not in families:
+                            family = family.group(1)
+                            specified_families = [x.strip().replace('"',
+                                '').replace("'", '') for x in family.split(',')]
+                            aliasing_ok = False
+                            for f in specified_families:
+                                bad_map[f] = families[0]
+                                if not aliasing_ok and f in families:
+                                    aliasing_ok = True
+
+                            if not aliasing_ok:
                                 prints('WARNING: Family aliasing not fully supported.')
-                                prints('\tDeclared family: %s not in actual families: %s'
+                                prints('\tDeclared family: %r not in actual families: %r'
                                         % (family, families))
                             else:
                                 prints('Loaded embedded font:', repr(family))

src/calibre/gui2/actions.py

@@ -21,6 +21,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
+from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \
     fetch_scheduled_recipe, generate_catalog
 from calibre.constants import preferred_encoding, filesystem_encoding, \
@@ -831,6 +832,23 @@ class EditMetadataAction(object): # {{{
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
         # }}}
 
+    def edit_device_collections(self, view, oncard=None):
+        model = view.model()
+        result = model.get_collections_with_ids()
+        compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        d = TagListEditor(self, tag_to_match=None, data=result, compare=compare)
+        d.exec_()
+        if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old ids
+            to_delete = d.to_delete # list of ids
+            for text in to_rename:
+                for old_id in to_rename[text]:
+                    model.rename_collection(old_id, new_name=unicode(text))
+            for item in to_delete:
+                model.delete_collection_using_id(item)
+            self.upload_collections(model.db, view=view, oncard=oncard)
+            view.reset()
+
     # }}}
 
 class SaveToDiskAction(object): # {{{

src/calibre/gui2/custom_column_widgets.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import re, sys
 from functools import partial
 
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
@@ -162,7 +162,6 @@ class DateTime(Base):
             val = qt_to_dt(val)
         return val
 
-
 class Comments(Base):
 
     def setup_ui(self, parent):
@@ -199,11 +198,7 @@ class Text(Base):
             w = EnComboBox(parent)
             w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
             w.setMinimumContentsLength(25)
-
-
-
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
-                w]
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
 
     def initialize(self, book_id):
         val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
@@ -222,7 +217,6 @@ class Text(Base):
             if idx is not None:
                 self.widgets[1].setCurrentIndex(idx)
 
-
     def setter(self, val):
         if self.col_metadata['is_multiple']:
             if not val:
@@ -241,6 +235,58 @@ class Text(Base):
             val = None
         return val
 
+class Series(Base):
+
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent))
+        w = QDoubleSpinBox(parent)
+        w.setRange(-100., float(sys.maxint))
+        w.setDecimals(2)
+        w.setSpecialValueText(_('Undefined'))
+        w.setSingleStep(1)
+        self.idx_widget=w
+        self.widgets.append(w)
+
+    def initialize(self, book_id):
+        val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
+        s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True)
+        if s_index is None:
+            s_index = 0.0
+        self.idx_widget.setValue(s_index)
+        self.initial_index = s_index
+        self.initial_val = val
+        val = self.normalize_db_val(val)
+        idx = None
+        for i, c in enumerate(self.all_values):
+            if c == val:
+                idx = i
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+        if idx is not None:
+            self.widgets[1].setCurrentIndex(idx)
+
+    def commit(self, book_id, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        s_index = self.idx_widget.value()
+        if val != self.initial_val or s_index != self.initial_index:
+            if s_index == 0.0:
+                if tweaks['series_index_auto_increment'] == 'next':
+                    s_index = self.db.get_next_cc_series_num_for(val,
+                                                             num=self.col_id)
+                else:
+                    s_index = None
+            self.db.set_custom(book_id, val, extra=s_index,
+                               num=self.col_id, notify=notify)
+
 widgets = {
         'bool' : Bool,
         'rating' : Rating,
@@ -249,6 +295,7 @@ widgets = {
         'datetime': DateTime,
         'text' : Text,
         'comments': Comments,
+        'series': Series,
 }
 
 def field_sort(y, z, x=None):
@@ -257,35 +304,63 @@ def field_sort(y, z, x=None):
     n2 = 'zzzzz' if m2['datatype'] == 'comments' else m2['name']
     return cmp(n1.lower(), n2.lower())
 
-def populate_single_metadata_page(left, right, db, book_id, parent=None):
+def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, parent=None):
+    def widget_factory(type, col):
+        if bulk:
+            w = bulk_widgets[type](db, col, parent)
+        else:
+            w = widgets[type](db, col, parent)
+        w.initialize(book_id)
+        return w
     x = db.custom_column_num_map
     cols = list(x)
     cols.sort(cmp=partial(field_sort, x=x))
+    count_non_comment = len([c for c in cols if x[c]['datatype'] != 'comments'])
+
+    layout.setColumnStretch(1, 10)
+    if two_column:
+        turnover_point = (count_non_comment+1)/2
+        layout.setColumnStretch(3, 10)
+    else:
+        # Avoid problems with multi-line widgets
+        turnover_point = count_non_comment + 1000
     ans = []
-    for i, col in enumerate(cols):
-        w = widgets[x[col]['datatype']](db, col, parent)
+    column = row = 0
+    for col in cols:
+        dt = x[col]['datatype']
+        if dt == 'comments':
+            continue
+        w = widget_factory(dt, col)
         ans.append(w)
-        w.initialize(book_id)
-        layout = left if i%2 == 0 else right
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            w.widgets[0].setBuddy(w.widgets[1])
-            for c, widget in enumerate(w.widgets):
-                layout.addWidget(widget, row, c)
+        for c in range(0, len(w.widgets), 2):
+            w.widgets[c].setBuddy(w.widgets[c+1])
+            layout.addWidget(w.widgets[c], row, column)
+            layout.addWidget(w.widgets[c+1], row, column+1)
+            row += 1
+        if row >= turnover_point:
+            column += 2
+            turnover_point = count_non_comment + 1000
+            row = 0
+    if not bulk: # Add the comments fields
+        column = 0
+        for col in cols:
+            dt = x[col]['datatype']
+            if dt != 'comments':
+                continue
+            w = widget_factory(dt, col)
+            ans.append(w)
+            layout.addWidget(w.widgets[0], row, column, 1, 2)
+            if two_column and column == 0:
+                column = 2
+                continue
+            column = 0
+            row += 1
     items = []
     if len(ans) > 0:
         items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
             QSizePolicy.Expanding))
-        left.addItem(items[-1], left.rowCount(), 0, 1, 1)
-        left.setRowStretch(left.rowCount()-1, 100)
-    if len(ans) > 1:
-         items.append(QSpacerItem(10, 100, QSizePolicy.Minimum,
-             QSizePolicy.Expanding))
-         right.addItem(items[-1], left.rowCount(), 0, 1, 1)
-         right.setRowStretch(right.rowCount()-1, 100)
-
+        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
+        layout.setRowStretch(layout.rowCount()-1, 100)
     return ans, items
 
 class BulkBase(Base):
@@ -342,6 +417,47 @@ class BulkRating(BulkBase, Rating):
 class BulkDateTime(BulkBase, DateTime):
     pass
 
+class BulkSeries(BulkBase):
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
+        self.idx_widget=QCheckBox(parent)
+        self.widgets.append(self.idx_widget)
+
+    def initialize(self, book_id):
+        self.idx_widget.setChecked(False)
+        for c in self.all_values:
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+
+    def commit(self, book_ids, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        update_indices = self.idx_widget.checkState()
+        if val != '':
+            for book_id in book_ids:
+                if update_indices:
+                    if tweaks['series_index_auto_increment'] == 'next':
+                        s_index = self.db.get_next_cc_series_num_for\
+                                                        (val, num=self.col_id)
+                    else:
+                        s_index = 1.0
+                else:
+                    s_index = self.db.get_custom_extra(book_id, num=self.col_id,
+                                                       index_is_id=True)
+                self.db.set_custom(book_id, val, extra=s_index,
+                                   num=self.col_id, notify=notify)
+
+    def process_each_book(self):
+        return True
+
 class RemoveTags(QWidget):
 
     def __init__(self, parent, values):
@@ -431,35 +547,5 @@ bulk_widgets = {
         'float': BulkFloat,
         'datetime': BulkDateTime,
         'text' : BulkText,
-}
-
-def populate_bulk_metadata_page(layout, db, book_ids, parent=None):
-    x = db.custom_column_num_map
-    cols = list(x)
-    cols.sort(cmp=partial(field_sort, x=x))
-    ans = []
-    for i, col in enumerate(cols):
-        dt = x[col]['datatype']
-        if dt == 'comments':
-            continue
-        w = bulk_widgets[dt](db, col, parent)
-        ans.append(w)
-        w.initialize(book_ids)
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            for c in range(0, len(w.widgets), 2):
-                w.widgets[c].setBuddy(w.widgets[c+1])
-                layout.addWidget(w.widgets[c], row, 0)
-                layout.addWidget(w.widgets[c+1], row, 1)
-                row += 1
-    items = []
-    if len(ans) > 0:
-        items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
-            QSizePolicy.Expanding))
-        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
-        layout.setRowStretch(layout.rowCount()-1, 100)
-
-    return ans, items
-
+        'series': BulkSeries,
+}

src/calibre/gui2/device.py

@@ -294,6 +294,11 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
+    def upload_collections(self, done, booklist, on_card):
+        return self.create_job(booklist.rebuild_collections, done,
+                               args=[booklist, on_card],
+                        description=_('Send collections to device'))
+
     def _upload_books(self, files, names, on_card=None, metadata=None):
         '''Upload books to device: '''
         return self.device.upload_books(files, names, on_card,
@@ -1228,6 +1233,19 @@ class DeviceMixin(object): # {{{
             return
         cp, fs = job.result
         self.location_view.model().update_devices(cp, fs)
+        # reset the views so that up-to-date info is shown. These need to be
+        # here because the sony driver updates collections in sync_booklists
+        self.memory_view.reset()
+        self.card_a_view.reset()
+        self.card_b_view.reset()
+
+    def _upload_collections(self, job):
+        if job.failed:
+            self.device_job_exception(job)
+
+    def upload_collections(self, booklist, view=None, oncard=None):
+        return self.device_manager.upload_collections(self._upload_collections,
+                                                       booklist, oncard)
 
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''

src/calibre/gui2/dialogs/config/add_save.py

@@ -45,6 +45,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.opt_swap_author_names.setChecked(prefs['swap_author_names'])
         self.opt_add_formats_to_existing.setChecked(prefs['add_formats_to_existing'])
+        self.preserve_user_collections.setChecked(prefs['preserve_user_collections'])
         help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
         self.save_template.initialize('save_to_disk', opts.template, help)
         self.send_template.initialize('send_to_device', opts.send_template, help)
@@ -71,6 +72,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         prefs['filename_pattern'] = pattern
         prefs['swap_author_names'] = bool(self.opt_swap_author_names.isChecked())
         prefs['add_formats_to_existing'] = bool(self.opt_add_formats_to_existing.isChecked())
+        prefs['preserve_user_collections'] = bool(self.preserve_user_collections.isChecked())
 
         return True
 

src/calibre/gui2/dialogs/config/add_save.ui

@@ -51,7 +51,7 @@
     <item row="2" column="0" colspan="2">
      <widget class="QCheckBox" name="opt_add_formats_to_existing">
       <property name="toolTip">
-       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added 
+       <string>If an existing book with a similar title and author is found that does not have the format being added, the format is added
 to the existing book, instead of creating a new entry. If the existing book already has the format, then it is silently ignored.
 
 Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;, &quot;an&quot;), punctuation, case, etc. Author match is exact.</string>
@@ -179,7 +179,31 @@ Title match ignores leading indefinite articles (&quot;the&quot;, &quot;a&quot;,
    </attribute>
    <layout class="QVBoxLayout" name="verticalLayout_2">
     <item>
-     <widget class="QLabel" name="label_4">
+     <widget class="QCheckBox" name="preserve_user_collections">
+      <property name="text">
+       <string>Preserve device collections.</string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_41">
+      <property name="text">
+       <string>If checked, collections will not be deleted even if a book with changed metadata is resent and the collection is not in the book's metadata. In addition, editing collections in the device view will be enabled. If unchecked, collections will be always reflect only the metadata in the calibre library.</string>
+      </property>
+      <property name="wordWrap">
+       <bool>true</bool>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_42">
+      <property name="text">
+       <string>  </string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_43">
       <property name="text">
        <string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Plugins</string>
       </property>

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -24,16 +24,19 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     2:{'datatype':'comments',
                         'text':_('Long text, like comments, not shown in the tag browser'),
                         'is_multiple':False},
-                    3:{'datatype':'datetime',
+                    3:{'datatype':'series',
+                        'text':_('Text column for keeping series-like information'),
+                        'is_multiple':False},
+                    4:{'datatype':'datetime',
                         'text':_('Date'), 'is_multiple':False},
-                    4:{'datatype':'float',
+                    5:{'datatype':'float',
                         'text':_('Floating point numbers'), 'is_multiple':False},
-                    5:{'datatype':'int',
+                    6:{'datatype':'int',
                         'text':_('Integers'), 'is_multiple':False},
-                    6:{'datatype':'rating',
+                    7:{'datatype':'rating',
                         'text':_('Ratings, shown with stars'),
                         'is_multiple':False},
-                    7:{'datatype':'bool',
+                    8:{'datatype':'bool',
                         'text':_('Yes/No'), 'is_multiple':False},
                 }
 

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -10,7 +10,7 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, \
     authors_to_string
-from calibre.gui2.custom_column_widgets import populate_bulk_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -44,15 +44,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.central_widget.tabBar().setVisible(False)
         else:
             self.create_custom_column_editors()
-
         self.exec_()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = QGridLayout()
-
-        self.custom_column_widgets, self.__cc_spacers = populate_bulk_metadata_page(
-                layout, self.db, self.ids, w)
+        self.custom_column_widgets, self.__cc_spacers = \
+            populate_metadata_page(layout, self.db, self.ids, parent=w,
+                                   two_column=False, bulk=True)
         w.setLayout(layout)
         self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets

src/calibre/gui2/dialogs/metadata_single.py

@@ -32,7 +32,7 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import qt_to_dt
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.dialogs.config.social import SocialMetadata
-from calibre.gui2.custom_column_widgets import populate_single_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class CoverFetcher(QThread):
 
@@ -420,23 +420,19 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
-        top_layout = QHBoxLayout()
-        top_layout.setSpacing(20)
-        left_layout = QGridLayout()
-        right_layout = QGridLayout()
-        top_layout.addLayout(left_layout)
-
-        self.custom_column_widgets, self.__cc_spacers = populate_single_metadata_page(
-                left_layout, right_layout, self.db, self.id, w)
-        top_layout.addLayout(right_layout)
-        sip.delete(w.layout())
-        w.setLayout(top_layout)
-        self.__custom_col_layouts = [top_layout, left_layout, right_layout]
+        layout = w.layout()
+        self.custom_column_widgets, self.__cc_spacers = \
+                    populate_metadata_page(layout, self.db, self.id,
+                                           parent=w, bulk=False, two_column=True)
+        self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets
         for i in range(len(ans)-1):
-            w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[-1])
-
-
+            if len(ans[i+1].widgets) == 2:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1])
+            else:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0])
+            for c in range(2, len(ans[i].widgets), 2):
+                w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1])
 
     def validate_isbn(self, isbn):
         isbn = unicode(isbn).strip()

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,54 +1,64 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from functools import partial
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
-from calibre.ebooks.metadata import title_sort
+
+class ListWidgetItem(QListWidgetItem):
+
+    def __init__(self, txt):
+        QListWidgetItem.__init__(self, txt)
+        self.old_value = txt
+        self.cur_value = txt
+
+    def data(self, role):
+        if role == Qt.DisplayRole:
+            if self.old_value != self.cur_value:
+                return _('%s (was %s)'%(self.cur_value, self.old_value))
+            else:
+                return self.cur_value
+        elif role == Qt.EditRole:
+            return self.cur_value
+        else:
+            return QListWidgetItem.data(self, role)
+
+    def setData(self, role, data):
+        if role == Qt.EditRole:
+            self.cur_value = data.toString()
+        QListWidgetItem.setData(self, role, data)
+
+    def text(self):
+        return self.cur_value
+
+    def setText(self, txt):
+        self.cur_value = txt
+        QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, db, tag_to_match, category):
+    def __init__(self, window, tag_to_match, data, compare):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
 
         self.to_rename = {}
         self.to_delete = []
-        self.db = db
         self.all_tags = {}
-        self.category = category
-        if category == 'tags':
-            result = db.get_tags_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        elif category == 'series':
-            result = db.get_series_with_ids()
-            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
-        elif category == 'publisher':
-            result = db.get_publishers_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        else: # should be a custom field
-            self.cc_label = None
-            if category in db.field_metadata:
-                self.cc_label = db.field_metadata[category]['label']
-                result = self.db.get_custom_items_with_ids(label=self.cc_label)
-            else:
-                result = []
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
 
-        for k,v in result:
+        for k,v in data:
             self.all_tags[v] = k
         for tag in sorted(self.all_tags.keys(), cmp=compare):
-            item = QListWidgetItem(tag)
+            item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)
 
-        items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
-        if len(items) == 1:
-            self.available_tags.setCurrentItem(items[0])
+        if tag_to_match is not None:
+            items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
+            if len(items) == 1:
+                self.available_tags.setCurrentItem(items[0])
 
         self.connect(self.delete_button,  SIGNAL('clicked()'), self.delete_tags)
         self.connect(self.rename_button,  SIGNAL('clicked()'), self.rename_tag)
@@ -62,13 +72,11 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 item.setText(self.item_before_editing.text())
                 return
         if item.text() != self.item_before_editing.text():
-            if item.text() in self.all_tags.keys() or item.text() in self.to_rename.keys():
-                error_dialog(self, _('Item already used'),
-                             _('The item %s is already used.')%(item.text())).exec_()
-                item.setText(self.item_before_editing.text())
-                return
             (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            self.to_rename[item.text()] = id
+            if item.text() not in self.to_rename:
+                self.to_rename[item.text()] = [id]
+            else:
+                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -99,30 +107,3 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             self.to_delete.append(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
-    def accept(self):
-        rename_func = None
-        if self.category == 'tags':
-            rename_func = self.db.rename_tag
-            delete_func = self.db.delete_tag_using_id
-        elif self.category == 'series':
-            rename_func = self.db.rename_series
-            delete_func = self.db.delete_series_using_id
-        elif self.category == 'publisher':
-            rename_func = self.db.rename_publisher
-            delete_func = self.db.delete_publisher_using_id
-        else:
-            rename_func = partial(self.db.rename_custom_item, label=self.cc_label)
-            delete_func = partial(self.db.delete_custom_item_using_id, label=self.cc_label)
-
-        work_done = False
-        if rename_func:
-            for text in self.to_rename:
-                work_done = True
-                rename_func(id=self.to_rename[text], new_name=unicode(text))
-            for item in self.to_delete:
-                work_done = True
-                delete_func(item)
-        if not work_done:
-            QDialog.reject(self)
-        else:
-            QDialog.accept(self)

src/calibre/gui2/init.py

@@ -226,17 +226,30 @@ class LibraryViewMixin(object): # {{{
                                         self.action_show_book_details,
                                         self.action_del,
                                         add_to_library = None,
+                                        edit_device_collections=None,
                                         similar_menu=similar_menu)
         add_to_library = (_('Add books to library'), self.add_books_from_device)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard=None))
         self.memory_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='carda'))
         self.card_a_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='cardb'))
         self.card_b_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
 
         self.library_view.files_dropped.connect(self.files_dropped, type=Qt.QueuedConnection)
         for func, args in [
@@ -249,9 +262,14 @@ class LibraryViewMixin(object): # {{{
                 getattr(view, func)(*args)
 
         self.memory_view.connect_dirtied_signal(self.upload_booklists)
+        self.memory_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard=None)
         self.card_a_view.connect_dirtied_signal(self.upload_booklists)
+        self.card_a_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='carda')
         self.card_b_view.connect_dirtied_signal(self.upload_booklists)
-
+        self.card_b_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='cardb')
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
         self.library_view.set_database(db)

src/calibre/gui2/library/models.py

@@ -16,11 +16,12 @@ from calibre.gui2 import NONE, config, UNDEFINED_QDATE
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
@@ -520,7 +521,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(', '.join(sorted(tags.split(','))))
             return None
 
-        def series(r, idx=-1, siix=-1):
+        def series_type(r, idx=-1, siix=-1):
             series = self.db.data[r][idx]
             if series:
                 idx = fmt_sidx(self.db.data[r][siix])
@@ -591,7 +592,7 @@ class BooksModel(QAbstractTableModel): # {{{
                                 idx=self.db.field_metadata['publisher']['rec_index'], mult=False),
                    'tags'     : functools.partial(tags,
                                 idx=self.db.field_metadata['tags']['rec_index']),
-                   'series'   : functools.partial(series,
+                   'series'   : functools.partial(series_type,
                                 idx=self.db.field_metadata['series']['rec_index'],
                                 siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
@@ -620,6 +621,9 @@ class BooksModel(QAbstractTableModel): # {{{
                                             bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
+            elif datatype == 'series':
+                self.dc[col] = functools.partial(series_type, idx=idx,
+                    siix=self.db.field_metadata.cc_series_index_column_for(col))
             else:
                 print 'What type is this?', col, datatype
         # build a index column to data converter map, to remove the string lookup in the data loop
@@ -681,6 +685,8 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_custom_column_data(self, row, colhead, value):
         typ = self.custom_columns[colhead]['datatype']
+        label=self.db.field_metadata.key_to_label(colhead)
+        s_index = None
         if typ in ('text', 'comments'):
             val = unicode(value.toString()).strip()
             val = val if val else None
@@ -702,9 +708,10 @@ class BooksModel(QAbstractTableModel): # {{{
                 if not val.isValid():
                     return False
                 val = qt_to_dt(val, as_utc=False)
-        self.db.set_custom(self.db.id(row), val,
-                           label=self.db.field_metadata.key_to_label(colhead),
-                           num=None, append=False, notify=True)
+        elif typ == 'series':
+            val, s_index = parse_series_string(self.db, label, value.toString())
+        self.db.set_custom(self.db.id(row), val, extra=s_index,
+                           label=label, num=None, append=False, notify=True)
         return True
 
     def setData(self, index, value, role):
@@ -850,6 +857,7 @@ class OnDeviceSearch(SearchQueryParser): # {{{
 class DeviceBooksModel(BooksModel): # {{{
 
     booklist_dirtied = pyqtSignal()
+    upload_collections = pyqtSignal(object)
 
     def __init__(self, parent):
         BooksModel.__init__(self, parent)
@@ -920,11 +928,12 @@ class DeviceBooksModel(BooksModel): # {{{
         if index.isValid() and self.editable:
             cname = self.column_map[index.column()]
             if cname in ('title', 'authors') or \
-                    (cname == 'collections' and self.db.supports_collections()):
+                    (cname == 'collections' and \
+                     self.db.supports_collections() and \
+                     prefs['preserve_user_collections']):
                 flags |= Qt.ItemIsEditable
         return flags
 
-
     def search(self, text, reset=True):
         if not text or not text.strip():
             self.map = list(range(len(self.db)))
@@ -970,8 +979,8 @@ class DeviceBooksModel(BooksModel): # {{{
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(self.db[x].device_collections)
-            y = ','.join(self.db[y].device_collections)
+            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []))).lower()
+            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []))).lower()
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1072,6 +1081,36 @@ class DeviceBooksModel(BooksModel): # {{{
                 res.append((r,b))
         return res
 
+    def get_collections_with_ids(self):
+        collections = set()
+        for book in self.db:
+            if book.device_collections is not None:
+                collections.update(set(book.device_collections))
+        self.collections = []
+        result = []
+        for i,collection in enumerate(collections):
+            result.append((i, collection))
+            self.collections.append(collection)
+        return result
+
+    def rename_collection(self, old_id, new_name):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+                if new_name not in book.device_collections:
+                    book.device_collections.append(new_name)
+
+    def delete_collection_using_id(self, old_id):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+
     def indices(self, rows):
         '''
         Return indices into underlying database from rows
@@ -1102,6 +1141,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
+                    tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
                     return QVariant(', '.join(tags))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
@@ -1144,14 +1184,18 @@ class DeviceBooksModel(BooksModel): # {{{
                 return False
             val = unicode(value.toString()).strip()
             idx = self.map[row]
+            if cname == 'collections':
+                tags = [i.strip() for i in val.split(',')]
+                tags = [t for t in tags if t]
+                self.db[idx].device_collections = tags
+                self.dataChanged.emit(index, index)
+                self.upload_collections.emit(self.db)
+                return True
+
             if cname == 'title' :
                 self.db[idx].title = val
             elif cname == 'authors':
                 self.db[idx].authors = string_to_authors(val)
-            elif cname == 'collections':
-                tags = [i.strip() for i in val.split(',')]
-                tags = [t for t in tags if t]
-                self.db[idx].device_collections = tags
             self.dataChanged.emit(index, index)
             self.booklist_dirtied.emit()
             done = True

src/calibre/gui2/library/views.py

@@ -15,7 +15,7 @@ from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
 from calibre.gui2.library import DEFAULT_SORT
 
@@ -347,7 +347,7 @@ class BooksView(QTableView): # {{{
                     self.setItemDelegateForColumn(cm.index(colhead), delegate)
                 elif cc['datatype'] == 'comments':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_comments_delegate)
-                elif cc['datatype'] == 'text':
+                elif cc['datatype'] in ('text', 'series'):
                     if cc['is_multiple']:
                         self.setItemDelegateForColumn(cm.index(colhead), self.tags_delegate)
                     else:
@@ -371,7 +371,8 @@ class BooksView(QTableView): # {{{
     # Context Menu {{{
     def set_context_menu(self, edit_metadata, send_to_device, convert, view,
                          save, open_folder, book_details, delete,
-                         similar_menu=None, add_to_library=None):
+                         similar_menu=None, add_to_library=None,
+                         edit_device_collections=None):
         self.setContextMenuPolicy(Qt.DefaultContextMenu)
         self.context_menu = QMenu(self)
         if edit_metadata is not None:
@@ -393,6 +394,10 @@ class BooksView(QTableView): # {{{
         if add_to_library is not None:
             func = partial(add_to_library[1], view=self)
             self.context_menu.addAction(add_to_library[0], func)
+        if edit_device_collections is not None:
+            func = partial(edit_device_collections[1], view=self)
+            self.edit_collections_menu = \
+                self.context_menu.addAction(edit_device_collections[0], func)
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
@@ -494,6 +499,13 @@ class DeviceBooksView(BooksView): # {{{
         self.setDragDropMode(self.NoDragDrop)
         self.setAcceptDrops(False)
 
+    def contextMenuEvent(self, event):
+        self.edit_collections_menu.setVisible(
+                            self._model.db.supports_collections() and \
+                            prefs['preserve_user_collections'])
+        self.context_menu.popup(event.globalPos())
+        event.accept()
+
     def set_database(self, db):
         self._model.set_database(db)
         self.restore_state()
@@ -505,6 +517,9 @@ class DeviceBooksView(BooksView): # {{{
     def connect_dirtied_signal(self, slot):
         self._model.booklist_dirtied.connect(slot)
 
+    def connect_upload_collections_signal(self, func=None, oncard=None):
+        self._model.upload_collections.connect(partial(func, view=self, oncard=oncard))
+
     def dropEvent(self, *args):
         error_dialog(self, _('Not allowed'),
         _('Dropping onto a device is not supported. First add the book to the calibre library.')).exec_()

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -75,10 +75,6 @@
 
 #include <QDebug>
 
-// uncomment this to enable bilinear filtering for texture mapping
-// gives much better rendering, at the cost of memory space
-// #define PICTUREFLOW_BILINEAR_FILTER
-
 // for fixed-point arithmetic, we need minimum 32-bit long
 // long long (64-bit) might be useful for multiplication and division
 typedef long PFreal;
@@ -376,7 +372,6 @@ private:
   int slideWidth;
   int slideHeight;
   int fontSize;
-  int zoom;
   int queueLength;
 
   int centerIndex;
@@ -401,6 +396,7 @@ private:
 
   void recalc(int w, int h);
   QRect renderSlide(const SlideInfo &slide, int alpha=256, int col1=-1, int col=-1);
+  QRect renderCenterSlide(const SlideInfo &slide);
   QImage* surface(int slideIndex);
   void triggerRender();
   void resetSlides();
@@ -414,7 +410,6 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_)
   slideWidth = 200;
   slideHeight = 200;
   fontSize = 10;
-  zoom = 100;
 
   centerIndex = 0;
   queueLength = queueLength_;
@@ -464,21 +459,6 @@ void PictureFlowPrivate::setSlideSize(QSize size)
   triggerRender();
 }
 
-int PictureFlowPrivate::zoomFactor() const
-{
-  return zoom;
-}
-
-void PictureFlowPrivate::setZoomFactor(int z)
-{
-  if(z <= 0)
-    return;
-
-  zoom = z;
-  recalc(buffer.width(), buffer.height());
-  triggerRender();
-}
-
 QImage PictureFlowPrivate::slide(int index) const
 {
   return slideImages->image(index);
@@ -554,7 +534,8 @@ void PictureFlowPrivate::resize(int w, int h)
   if (w < 10) w = 10;
   if (h < 10) h = 10;
   slideHeight = int(float(h)/REFLECTION_FACTOR);
-  slideWidth = int(float(slideHeight) * 2/3.);
+  slideWidth = int(float(slideHeight) * 3./4.);
+  //qDebug() << slideHeight << "x" << slideWidth;
   fontSize = MAX(int(h/15.), 12);
   recalc(w, h);
   resetSlides();
@@ -595,15 +576,12 @@ void PictureFlowPrivate::resetSlides()
   }
 }
 
-#define BILINEAR_STRETCH_HOR 4
-#define BILINEAR_STRETCH_VER 4
-
 static QImage prepareSurface(QImage img, int w, int h)
 {
   Qt::TransformationMode mode = Qt::SmoothTransformation;
-  img = img.scaled(w, h, Qt::IgnoreAspectRatio, mode);
+  img = img.scaled(w, h, Qt::KeepAspectRatioByExpanding, mode);
 
-  // slightly larger, to accomodate for the reflection
+  // slightly larger, to accommodate for the reflection
   int hs = int(h * REFLECTION_FACTOR);
   int hofs = 0;
 
@@ -633,12 +611,6 @@ static QImage prepareSurface(QImage img, int w, int h)
       result.setPixel(h+hofs+y, x, qRgb(r, g, b));
     }
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int hh = BILINEAR_STRETCH_VER*hs;
-  int ww = BILINEAR_STRETCH_HOR*w;
-  result = result.scaled(hh, ww, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
-#endif
-
   return result;
 }
 
@@ -699,8 +671,12 @@ void PictureFlowPrivate::render()
 
   int nleft = leftSlides.count();
   int nright = rightSlides.count();
+  QRect r;
 
-  QRect r = renderSlide(centerSlide);
+  if (step == 0) 
+      r = renderCenterSlide(centerSlide);
+  else
+      r = renderSlide(centerSlide);
   int c1 = r.left();
   int c2 = r.right();
 
@@ -813,7 +789,23 @@ static inline uint BYTE_MUL_RGB16_32(uint x, uint a) {
     return t;
 }
 
+QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) {
+  QImage* src = surface(slide.slideIndex);
+  if(!src)
+    return QRect();
 
+  int sw = src->height();
+  int sh = src->width();
+  int h = buffer.height();
+  QRect rect(buffer.width()/2 - sw/2, 0, sw, h-1);
+  int left = rect.left();
+
+  for(int x = 0; x < sh-1; x++)
+    for(int y = 0; y < sw; y++)
+      buffer.setPixel(left + y, 1+x, src->pixel(x, y));
+
+  return rect;
+}
 // Renders a slide to offscreen buffer. Returns a rect of the rendered area.
 // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent
 // col1 and col2 limit the column for rendering.
@@ -826,13 +818,8 @@ int col1, int col2)
 
   QRect rect(0, 0, 0, 0);  
   
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int sw = src->height() / BILINEAR_STRETCH_HOR;
-  int sh = src->width() / BILINEAR_STRETCH_VER;
-#else
   int sw = src->height();
   int sh = src->width();
-#endif
   int h = buffer.height();
   int w = buffer.width();
 
@@ -848,7 +835,7 @@ int col1, int col2)
   col1 = qMin(col1, w-1);
   col2 = qMin(col2, w-1);
 
-  int distance = h * 100 / zoom;
+  int distance = h;
   PFreal sdx = fcos(slide.angle);
   PFreal sdy = fsin(slide.angle);
   PFreal xs = slide.cx - slideWidth * sdx/2;
@@ -878,15 +865,9 @@ int col1, int col2)
     PFreal hitx = fmul(dist, rays[x]);
     PFreal hitdist = fdiv(hitx - slide.cx, sdx);
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int column = sw*BILINEAR_STRETCH_HOR/2 + (hitdist*BILINEAR_STRETCH_HOR >> PFREAL_SHIFT);
-    if(column >= sw*BILINEAR_STRETCH_HOR)
-      break;
-#else
     int column = sw/2 + (hitdist >> PFREAL_SHIFT);
     if(column >= sw)
       break;
-#endif
     if(column < 0)
       continue;
 
@@ -901,13 +882,8 @@ int col1, int col2)
     QRgb565* pixel2 = (QRgb565*)(buffer.scanLine(y2)) + x;
     int pixelstep = pixel2 - pixel1;
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int center = (sh*BILINEAR_STRETCH_VER/2);
-    int dy = dist*BILINEAR_STRETCH_VER / h;
-#else
     int center = sh/2;
     int dy = dist / h;
-#endif
     int p1 = center*PFREAL_ONE - dy/2;
     int p2 = center*PFREAL_ONE + dy/2;
 
@@ -1155,16 +1131,6 @@ void PictureFlow::setSlideSize(QSize size)
   d->setSlideSize(size);
 }
 
-int PictureFlow::zoomFactor() const
-{
-  return d->zoomFactor();
-}
-
-void PictureFlow::setZoomFactor(int z)
-{
-  d->setZoomFactor(z);
-}
-
 QImage PictureFlow::slide(int index) const
 {
   return d->slide(index);

src/calibre/gui2/pictureflow/pictureflow.h

@@ -91,7 +91,6 @@ Q_OBJECT
 
   Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide)
   Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize)
-  Q_PROPERTY(int zoomFactor READ zoomFactor WRITE setZoomFactor)
 
 public:
   /*!
@@ -120,16 +119,6 @@ public:
   */  
   void setSlideSize(QSize size);
 
-  /*!
-    Sets the zoom factor (in percent).
-  */ 
-  void setZoomFactor(int zoom);
-
-  /*!
-    Returns the zoom factor (in percent).
-  */
-  int zoomFactor() const;
-
   /*!
     Clears any caches held to free up memory
   */

src/calibre/gui2/pictureflow/pictureflow.sip

@@ -40,10 +40,6 @@ public :
 
   void setSlideSize(QSize size);
 
-  void setZoomFactor(int zoom);
-
-  int zoomFactor() const;
-
   void clearCaches();
 
   virtual QImage slide(int index) const;

src/calibre/gui2/tag_view.py

@@ -15,6 +15,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
                      QPushButton, QWidget, QItemDelegate
 
+from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
 from calibre.library.field_metadata import TagsIcons
@@ -680,9 +681,50 @@ class TagBrowserMixin(object): # {{{
             self.tags_view.recount()
 
     def do_tags_list_edit(self, tag, category):
-        d = TagListEditor(self, self.library_view.model().db, tag, category)
+        db=self.library_view.model().db
+        if category == 'tags':
+            result = db.get_tags_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        elif category == 'series':
+            result = db.get_series_with_ids()
+            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
+        elif category == 'publisher':
+            result = db.get_publishers_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        else: # should be a custom field
+            cc_label = None
+            if category in db.field_metadata:
+                cc_label = db.field_metadata[category]['label']
+                result = self.db.get_custom_items_with_ids(label=cc_label)
+            else:
+                result = []
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+
+        d = TagListEditor(self, tag_to_match=tag, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            rename_func = None
+            if category == 'tags':
+                rename_func = db.rename_tag
+                delete_func = db.delete_tag_using_id
+            elif category == 'series':
+                rename_func = db.rename_series
+                delete_func = db.delete_series_using_id
+            elif category == 'publisher':
+                rename_func = db.rename_publisher
+                delete_func = db.delete_publisher_using_id
+            else:
+                rename_func = partial(db.rename_custom_item, label=cc_label)
+                delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
+            if rename_func:
+                for text in to_rename:
+                        for old_id in to_rename[text]:
+                            rename_func(old_id, new_name=unicode(text))
+                for item in to_delete:
+                    delete_func(item)
+
             # Clean up everything, as information could have changed for many books.
             self.library_view.model().refresh()
             self.tags_view.set_new_model()

src/calibre/gui2/ui.py

@@ -473,6 +473,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
             self.search_restriction.setEnabled(False)
             for action in list(self.delete_menu.actions())[1:]:
                 action.setEnabled(False)
+            # Reset the view in case something changed while it was invisible
+            self.current_view().reset()
         self.set_number_of_books_shown()
 
 

src/calibre/library/caches.py

@@ -401,7 +401,8 @@ class ResultCache(SearchQueryParser):
             for x in self.field_metadata:
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
-                    if self.field_metadata[x]['datatype'] not in ['text', 'comments']:
+                    if self.field_metadata[x]['datatype'] not in \
+                                                ['text', 'comments', 'series']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
                     is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@@ -580,16 +581,18 @@ class ResultCache(SearchQueryParser):
             self.sort(field, ascending)
         self._map_filtered = list(self._map)
 
-    def seriescmp(self, x, y):
-        sidx = self.FIELD_MAP['series']
+    def seriescmp(self, sidx, siidx, x, y, library_order=None):
         try:
-            ans = cmp(title_sort(self._data[x][sidx].lower()),
-                    title_sort(self._data[y][sidx].lower()))
+            if library_order:
+                ans = cmp(title_sort(self._data[x][sidx].lower()),
+                                title_sort(self._data[y][sidx].lower()))
+            else:
+                ans = cmp(self._data[x][sidx].lower(),
+                                                self._data[y][sidx].lower())
         except AttributeError: # Some entries may be None
             ans = cmp(self._data[x][sidx], self._data[y][sidx])
         if ans != 0: return ans
-        sidx = self.FIELD_MAP['series_index']
-        return cmp(self._data[x][sidx], self._data[y][sidx])
+        return cmp(self._data[x][siidx], self._data[y][siidx])
 
     def cmp(self, loc, x, y, asstr=True, subsort=False):
         try:
@@ -617,18 +620,27 @@ class ResultCache(SearchQueryParser):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
-        if self.field_metadata[field]['is_custom']:
-            as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-            field = self.field_metadata[field]['colnum']
 
         if self.first_sort:
             subsort = True
             self.first_sort = False
-        fcmp = self.seriescmp \
-                if field == 'series' and \
-                   tweaks['title_series_sorting'] == 'library_order' \
-                else \
-                   functools.partial(self.cmp, self.FIELD_MAP[field],
+        if self.field_metadata[field]['is_custom']:
+            if self.field_metadata[field]['datatype'] == 'series':
+                fcmp = functools.partial(self.seriescmp,
+                    self.field_metadata[field]['rec_index'],
+                    self.field_metadata.cc_series_index_column_for(field),
+                    library_order=tweaks['title_series_sorting'] == 'library_order')
+            else:
+                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
+                field = self.field_metadata[field]['colnum']
+                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                                     subsort=subsort, asstr=as_string)
+        elif field == 'series':
+            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
+                self.FIELD_MAP['series_index'],
+                library_order=tweaks['title_series_sorting'] == 'library_order')
+        else:
+            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
                                      subsort=subsort, asstr=as_string)
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]

src/calibre/library/cli.py

@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO
+import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
 from calibre import terminal_controller, preferred_encoding, prints
-from calibre.utils.config import OptionParser, prefs
+from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
@@ -680,9 +680,31 @@ def command_catalog(args, dbpath):
 
 # end of GR additions
 
+def parse_series_string(db, label, value):
+    val = unicode(value).strip()
+    s_index = None
+    pat = re.compile(r'\[([.0-9]+)\]')
+    match = pat.search(val)
+    if match is not None:
+        val = pat.sub('', val).strip()
+        s_index = float(match.group(1))
+    elif val:
+        if tweaks['series_index_auto_increment'] == 'next':
+            s_index = db.get_next_cc_series_num_for(val, label=label)
+        else:
+            s_index = 1.0
+    return val, s_index
+
 def do_set_custom(db, col, id_, val, append):
-    db.set_custom(id_, val, label=col, append=append)
-    prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
+    if db.custom_column_label_map[col]['datatype'] == 'series':
+        val, s_index = parse_series_string(db, col, val)
+        db.set_custom(id_, val, extra=s_index, label=col, append=append)
+        prints('Data set to: %r[%4.2f]'%
+               (db.get_custom(id_, label=col, index_is_id=True),
+                db.get_custom_extra(id_, label=col, index_is_id=True)))
+    else:
+        db.set_custom(id_, val, label=col, append=append)
+        prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
 
 def set_custom_option_parser():
     parser = get_parser(_(

src/calibre/library/custom_columns.py

@@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import json
 from functools import partial
+from math import floor
 
 from calibre import prints
 from calibre.constants import preferred_encoding
@@ -16,7 +17,7 @@ from calibre.utils.date import parse_date
 class CustomColumns(object):
 
     CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
-        'int', 'float', 'bool'])
+        'int', 'float', 'bool', 'series'])
 
     def custom_table_names(self, num):
         return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@@ -137,7 +138,8 @@ class CustomColumns(object):
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
                 'datetime' : adapt_datetime,
-                'text':adapt_text
+                'text':adapt_text,
+                'series':adapt_text
         }
 
         # Create Tag Browser categories for custom columns
@@ -171,6 +173,19 @@ class CustomColumns(object):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
 
+    def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        # add future datatypes with an extra column here
+        if data['datatype'] not in ['series']:
+            return None
+        ign,lt = self.custom_table_names(data['num'])
+        idx = idx if index_is_id else self.id(idx)
+        return self.conn.get('''SELECT extra FROM %s
+                                WHERE book=?'''%lt, (idx,), all=False)
+
     # convenience methods for tag editing
     def get_custom_items_with_ids(self, label=None, num=None):
         if label is not None:
@@ -220,6 +235,28 @@ class CustomColumns(object):
             self.conn.commit()
     # end convenience methods
 
+    def get_next_cc_series_num_for(self, series, label=None, num=None):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        if data['datatype'] != 'series':
+            return None
+        table, lt = self.custom_table_names(data['num'])
+        # get the id of the row containing the series string
+        series_id = self.conn.get('SELECT id from %s WHERE value=?'%table,
+                                                        (series,), all=False)
+        if series_id is None:
+            return 1.0
+        # get the label of the associated series number table
+        series_num = self.conn.get('''
+                SELECT MAX({lt}.extra) FROM {lt}
+                WHERE {lt}.book IN (SELECT book FROM {lt} where value=?)
+                '''.format(lt=lt), (series_id,), all=False)
+        if series_num is None:
+            return 1.0
+        return floor(series_num+1)
+
     def all_custom(self, label=None, num=None):
         if label is not None:
             data = self.custom_column_label_map[label]
@@ -271,9 +308,8 @@ class CustomColumns(object):
             self.conn.commit()
         return changed
 
-
-
-    def set_custom(self, id_, val, label=None, num=None, append=False, notify=True):
+    def set_custom(self, id_, val, label=None, num=None,
+                   append=False, notify=True, extra=None):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:
@@ -317,10 +353,17 @@ class CustomColumns(object):
                             'INSERT INTO %s(value) VALUES(?)'%table, (x,)).lastrowid
                 if not self.conn.get(
                     'SELECT book FROM %s WHERE book=? AND value=?'%lt,
-                    (id_, xid), all=False):
-                    self.conn.execute(
-                        'INSERT INTO %s(book, value) VALUES (?,?)'%lt,
-                            (id_, xid))
+                                                        (id_, xid), all=False):
+                    if data['datatype'] == 'series':
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value, extra)
+                               VALUES (?,?,?)'''%lt, (id_, xid, extra))
+                        self.data.set(id_, self.FIELD_MAP[data['num']]+1,
+                                      extra, row_is_id=True)
+                    else:
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value)
+                                VALUES (?,?)'''%lt, (id_, xid))
             self.conn.commit()
             nval = self.conn.get(
                     'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'],
@@ -370,6 +413,9 @@ class CustomColumns(object):
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}
                 '''.format(query=query%table, lt=lt, table=table, num=data['num'])
+                if data['datatype'] == 'series':
+                    line += ''',(SELECT extra FROM {lt} WHERE {lt}.book=books.id)
+                        custom_index_{num}'''.format(lt=lt, num=data['num'])
             else:
                 line = '''
                 (SELECT value FROM {table} WHERE book=books.id) custom_{num}
@@ -393,7 +439,7 @@ class CustomColumns(object):
 
         if datatype in ('rating', 'int'):
             dt = 'INT'
-        elif datatype in ('text', 'comments'):
+        elif datatype in ('text', 'comments', 'series'):
             dt = 'TEXT'
         elif datatype in ('float',):
             dt = 'REAL'
@@ -404,6 +450,10 @@ class CustomColumns(object):
         collate = 'COLLATE NOCASE' if dt == 'TEXT' else ''
         table, lt = self.custom_table_names(num)
         if normalized:
+            if datatype == 'series':
+                s_index = 'extra REAL,'
+            else:
+                s_index = ''
             lines = [
                 '''\
                 CREATE TABLE %s(
@@ -419,8 +469,9 @@ class CustomColumns(object):
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     book INTEGER NOT NULL,
                     value INTEGER NOT NULL,
+                    %s
                     UNIQUE(book, value)
-                    );'''%lt,
+                    );'''%(lt, s_index),
 
                 'CREATE INDEX %s_aidx ON %s (value);'%(lt,lt),
                 'CREATE INDEX %s_bidx ON %s (book);'%(lt,lt),
@@ -468,7 +519,7 @@ class CustomColumns(object):
                           ratings as r
                      WHERE {lt}.value={table}.id and bl.book={lt}.book and
                            r.id = bl.rating and r.rating <> 0) avg_rating,
-                     value AS sort
+                    value AS sort
                 FROM {table};
 
                 CREATE VIEW tag_browser_filtered_{table} AS SELECT
@@ -483,7 +534,7 @@ class CustomColumns(object):
                      WHERE {lt}.value={table}.id AND bl.book={lt}.book AND
                            r.id = bl.rating AND r.rating <> 0 AND
                            books_list_filter(bl.book)) avg_rating,
-                     value AS sort
+                    value AS sort
                 FROM {table};
 
                 '''.format(lt=lt, table=table),

src/calibre/library/database2.py

@@ -237,6 +237,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         self.custom_column_num_map[col]['label'],
                                         base,
                                         prefer_custom=True)
+            if self.custom_column_num_map[col]['datatype'] == 'series':
+                # account for the series index column. Field_metadata knows that
+                # the series index is one larger than the series. If you change
+                # it here, be sure to change it there as well.
+                self.FIELD_MAP[str(col)+'_s_index'] = base = base+1
 
         self.FIELD_MAP['cover'] = base+1
         self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
@@ -777,6 +782,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         icon=icon, tooltip=tooltip)
                                     for r in data if item_not_zero_func(r)]
 
+        # Needed for legacy databases that have multiple ratings that
+        # map to n stars
+        for r in categories['rating']:
+            for x in categories['rating']:
+                if r.name == x.name and r.id != x.id:
+                    r.count = r.count + x.count
+                    categories['rating'].remove(x)
+                    break
+
         # We delayed computing the standard formats category because it does not
         # use a view, but is computed dynamically
         categories['formats'] = []

src/calibre/library/field_metadata.py

@@ -81,7 +81,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
-                           'datatype':'text',
+                           'datatype':'series',
                            'is_multiple':None,
                            'kind':'field',
                            'name':_('Series'),
@@ -398,6 +398,8 @@ class FieldMetadata(dict):
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
 
+    def cc_series_index_column_for(self, key):
+        return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
         if label in self._tb_cats:

src/calibre/manual/faq.rst

@@ -7,7 +7,7 @@ Frequently Asked Questions
 
 .. contents:: Contents
   :depth: 1
-  :local: 
+  :local:
 
 E-book Format Conversion
 -------------------------
@@ -30,7 +30,7 @@ It can convert every input format in the following list, to every output format.
 
 What are the best source formats to convert?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF 
+In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF
 
 Why does the PDF conversion lose some images/tables?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -40,7 +40,7 @@ are also represented as vector diagrams, thus they cannot be extracted.
 How do I convert a collection of HTML files in a specific order?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 In order to convert a collection of HTML files in a specific oder, you have to create a table of contents file. That is, another HTML file that contains links to all the other files in the desired order. Such a file looks like::
- 
+
    <html>
       <body>
         <h1>Table of Contents</h1>
@@ -60,16 +60,16 @@ Then just add this HTML file to the GUI and use the convert button to create you
 
 How do I convert my file containing non-English characters, or smart quotes?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-There are two aspects to this problem: 
+There are two aspects to this problem:
   1. Knowing the encoding of the source file: |app| tries to guess what character encoding your source files use, but often, this is impossible, so you need to tell it what encoding to use. This can be done in the GUI via the :guilabel:`Input character encoding` field in the :guilabel:`Look & Feel` section. The command-line tools all have an :option:`--input-encoding` option.
-  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8). 
-  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader. 
+  2. When adding HTML files to |app|, you may need to tell |app| what encoding the files are in. To do this go to Preferences->Plugins->File Type plugins and customize the HTML2Zip plugin, telling it what encoding your HTML files are in. Now when you add HTML files to |app| they will be correctly processed. HTML files from different sources often have different encodings, so you may have to change this setting repeatedly. A common encoding for many files from the web is ``cp1252`` and I would suggest you try that first. Note that when converting HTML files, leave the input encoding setting mentioned above blank. This is because the HTML2ZIP plugin automatically converts the HTML files to a standard encoding (utf-8).
+  3. Embedding fonts: If you are generating an LRF file to read on your SONY Reader, you are limited by the fact that the Reader only supports a few non-English characters in the fonts it comes pre-loaded with. You can work around this problem by embedding a unicode-aware font that supports the character set your file uses into the LRF file. You should embed atleast a serif and a sans-serif font. Be aware that embedding fonts significantly slows down page-turn speed on the reader.
 
 
 How do I use some of the advanced features of the conversion tools?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features: 
-  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_ 
+ You can get help on any individual feature of the converters by mousing over it in the GUI or running ``ebook-convert dummy.html .epub -h`` at a terminal. A good place to start is to look at the following demo files that demonstrate some of the advanced features:
+  * `html-demo.zip <http://calibre-ebook.com/downloads/html-demo.zip>`_
 
 
 Device Integration
@@ -95,12 +95,39 @@ We just need some information from you:
     device supports SD cards, insert them. Then connect your device. In calibre go to Preferences->Advanced
     and click the "Debug device detection" button. This will create some debug output. Copy it to a file
     and repeat the process, this time with your device disconnected.
-  * Send both the above outputs to us with the other information and we will write a device driver for your 
+  * Send both the above outputs to us with the other information and we will write a device driver for your
     device.
 
 Once you send us the output for a particular operating system, support for the device in that operating system
 will appear in the next release of |app|.
 
+How does |app| manage collections on my SONY reader?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When |app| connects with the device, it retrieves all collections for the books on the device. The collections
+of which books are members are shown on the device view.
+
+When you send a book to the device, |app| will add the book to collections based on the metadata for that book. By
+default, collections are created from tags and series. You can control what metadata is used by going to
+Preferences->Plugins->Device Interface plugins and customizing the SONY device interface plugin. If you remove all
+values, |app| will not add the book to any collection.
+
+Collection management is largely controlled by 'Preserve device collections' found at Preferences->Add/Save->Sending
+to device. If checked (the default), managing collections is left to the user; |app| will not delete already
+existing collections for a book on your device when you resend the book to the device, but |app| will add the book to
+collections if necessary.  To ensure that the collections for a book are based only on current |app| metadata, first
+delete the books from the device, then resend the books.  You can edit collections directly on the device view by
+double-clicking or right-clicking in the collections column.
+
+If 'Preserve device collections' is not checked, then |app| will manage collections. Collections will be built using
+|app| metadata exclusively.  Sending a book to the device will correct the collections for that book so its
+collections exactly match the book's metadata. Collections are added and deleted as necessary.  Editing collections on
+the device pane is not permitted, because collections not in the metadata will be removed automatically.
+
+In summary, check 'Preserve device collections' if you want to manage collections yourself.  Collections for a book
+will never be removed by |app|, but can be removed by you by editing on the device view.  Uncheck 'Preserve device
+collections' if you want |app| to manage the collections, adding books to and removing books from collections as
+needed.  
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -130,7 +157,7 @@ simplest is to simply re-name the executable file that launches the library prog
 Can I use the collections feature of the SONY reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| has full support for collections. When you add tags to a book's metadata, those tags are turned into collections when you upload the book to the SONY reader. Also, the series information is automatically
-turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does. 
+turned into a collection on the reader. Note that the PRS-500 does not support collections for books stored on the SD card. The PRS-505 does.
 
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -139,7 +166,7 @@ The easiest way to browse your |app| collection on your Apple device (iPad/iPhon
 
 First perform the following steps in |app|
 
-  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General) 
+  * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General)
   * Set the output profile to iPad (this will work for iPhone/iPods as well), under Preferences->Conversion->Page Setup
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
@@ -171,7 +198,7 @@ Can I access my |app| books using the web browser in my Kindle or other reading
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 |app| has a *Content Server* that exports the books in |app| as a web page. You can turn it on under
-Preferences->Content Server. Then just point the web browser on your device to the computer running 
+Preferences->Content Server. Then just point the web browser on your device to the computer running
 the Content Server and you will be able to browse your book collection. For example, if the computer running
 the server has IP address 63.45.128.5, in the browser, you would type::
 
@@ -190,8 +217,8 @@ The most likely cause of this is your antivirus program. Try temporarily disabli
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests:: 
-    
+|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests::
+
     grep SYSFS_DEPRECATED /boot/config-`uname -r`
 
 You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``.
@@ -227,7 +254,7 @@ Now this makes it very easy to find for example all science fiction books by Isa
 
     ReadStatus -> Genre -> Author -> Series
 
-In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search. 
+In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search.
 
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -235,7 +262,7 @@ Why doesn't |app| have a column for foo?
 
 How do I move my |app| library from one computer to another?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. 
+Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking Preferences. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder.
 
 Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to Preferences->Advanced and click the Check database integrity button. It will warn you about missing files, if any, which you should then transfer by hand.
 
@@ -246,11 +273,11 @@ Content From The Web
   :depth: 1
   :local:
 
-My downloaded news content causes the reader to reset. 
+My downloaded news content causes the reader to reset.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 This is a bug in the SONY firmware. The problem can be mitigated by switching the output format to EPUB
-in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software 
-to transfer the files to the reader. The SONY software pre-paginates the LRF file, 
+in the configuration dialog. Alternatively, you can use the LRF output format and use the SONY software
+to transfer the files to the reader. The SONY software pre-paginates the LRF file,
 thereby reducing the number of resets.
 
 I obtained a recipe for a news site as a .py file from somewhere, how do I use it?
@@ -285,7 +312,7 @@ Take your pick:
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts founf on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory. 
+|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts found on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
 
 |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -308,6 +335,10 @@ Post any output you see in a help message on the `Forum <http://www.mobileread.c
 |app| is not starting on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+One common cause of failures on OS X is the use of accessibility technologies that are incompatible with the graphics toolkit |app| uses.
+Try turning off VoiceOver if you have it on. Also go to System Preferences->System->Universal Access and turn off the setting for enabling
+access for assistive devices in all the tabs.
+
 You can obtain debug output about why |app| is not starting by running `Console.app`. Debug output will
 be printed to it. If the debug output contains a line that looks like::
 
@@ -317,9 +348,9 @@ then the problem is probably a corrupted font cache. You can clear the cache by
 `instructions <http://www.macworld.com/article/139383/2009/03/fontcacheclear.html>`_. If that doesn't
 solve it, look for a corrupted font file on your system, in ~/Library/Fonts or the like.
 
-
 My antivirus program claims |app| is a virus/trojan?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Your antivirus program is wrong. |app| is a completely open source product. You can actually browse the source code yourself (or hire someone to do it for you) to verify that it is not a virus. Please report the false identification to whatever company you buy your antivirus software from. If the antivirus program is preventing you from downloading/installing |app|, disable it temporarily, install |app| and then re-enable it.
 
 How do I use purchased EPUB books with |app|?
@@ -329,8 +360,8 @@ Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. Thi
 
 I want some feature added to |app|. What can I do?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-You have two choices: 
- 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_. 
+You have two choices:
+ 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development <http://calibre-ebook.com/get-involved>`_.
  2. `Open a ticket <http://bugs.calibre-ebook.com/newticket>`_ (you have to register and login first) and hopefully I will find the time to implement your feature.
 
 Can I include |app| on a CD to be distributed with my product/magazine?

src/calibre/utils/config.py

@@ -698,6 +698,8 @@ def _prefs():
     # calibre server can execute searches
     c.add_opt('saved_searches', default={}, help=_('List of named saved searches'))
     c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
+    c.add_opt('preserve_user_collections', default=True,
+        help=_('Preserve all collections even if not in library metadata.'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c