Pull from Trunk

resources/recipes/china_press.recipe

@@ -0,0 +1,71 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277228948(BasicNewsRecipe):
+    title          = u'China Press USA'
+    oldest_article = 7
+    max_articles_per_feed = 100
+
+    __author__            = 'rty'
+    __version__            = '1.0'
+    language = 'zh_CN'
+    pubisher  = 'www.chinapressusa.com'
+    description           = 'Overseas Chinese Network Newspaper in the USA'
+    category              = 'News in Chinese, USA'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    #encoding               = 'GB2312'
+    encoding               = 'UTF-8'
+    conversion_options = {'linearize_tables':True}
+    masthead_url ='http://www.chinapressusa.com/common/images/logo.gif'
+    extra_css = '''
+             @font-face { font-family: "DroidFont", serif, sans-serif;  src: url(res:///system/fonts/DroidSansFallback.ttf); }\n
+             body {
+                  margin-right: 8pt;
+                  font-family: 'DroidFont', serif;}
+              h1  {font-family: 'DroidFont', serif, sans-serif}
+            .show {font-family: 'DroidFont', serif, sans-serif}
+            '''
+    feeds          = [
+	(u'\u65b0\u95fb\u9891\u9053', u'http://news.uschinapress.com/news.xml'),
+	(u'\u534e\u4eba\u9891\u9053', u'http://chinese.uschinapress.com/chinese.xml'),
+	(u'\u8bc4\u8bba\u9891\u9053', u'http://review.uschinapress.com/review.xml'),
+	]
+    keep_only_tags = [
+                              dict(name='div', attrs={'class':'show'}),
+                               ]
+    remove_tags = [
+     #               dict(name='table', attrs={'class':'xle'}),
+                    dict(name='div', attrs={'class':'time'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='div', attrs={'class':'bank17'}),
+         #         dict(name='a', attrs={'class':'ab12'}),
+                         ]
+
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('div',attrs={'id':'displaypagenum'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'show'})
+           for it in texttag.findAll(style=True):
+               del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Language" content="zh-CN"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)
+
+        for item in soup.findAll(style=True):
+            del item['style']
+        self.append_page(soup, soup.body, 3)
+        pager = soup.find('div',attrs={'id':'displaypagenum'})
+        if pager:
+           pager.extract()
+        return soup

resources/recipes/london_free_press.recipe

@@ -0,0 +1,38 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonFreePress(BasicNewsRecipe):
+    title          = u'London Free Press'
+    __author__            = 'rty'
+    oldest_article = 4
+    max_articles_per_feed = 100
+
+    pubisher  = 'lfpress.com'
+    description           = 'Ontario Canada Newspaper'
+    category              = 'News, Ontario, Canada'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'en_CA'
+    encoding               = 'utf-8'
+    conversion_options = {'linearize_tables':True}
+
+    feeds          = [
+                    (u'News', u'http://www.lfpress.com/news/rss.xml'),
+                    (u'Comment', u'http://www.lfpress.com/comment/rss.xml'),
+                    (u'Entertainment', u'http://www.lfpress.com/entertainment/rss.xml '),
+                    (u'Money', u'http://www.lfpress.com/money/rss.xml '),
+                    (u'Life', u'http://www.lfpress.com/life/rss.xml '),
+                    (u'Sports', u'http://www.lfpress.com/sports/rss.xml ')
+                        ]
+
+    keep_only_tags = [
+                              dict(name='div', attrs={'id':'article'}),
+                               ]
+    remove_tags = [
+                    dict(name='div', attrs={'id':'commentsBottom'}),
+                    dict(name='div', attrs={'class':['leftBox','bottomBox clear']}),
+   	dict(name='ul', attrs={'class':'tabs dl contentSwap'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='div', attrs={'class':'bottomBox clear'}),
+                         ]

resources/recipes/losservatoreromano_it.recipe

@@ -0,0 +1,48 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.vatican.va/news_services/or/or_quo
+'''
+
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LOsservatoreRomano_it(BasicNewsRecipe):
+    title                = "L'Osservatore Romano"
+    __author__           = 'Darko Miletic'
+    description          = 'Quiornale quotidiano, politico, religioso del Vaticano'
+    publisher            = 'La Santa Sede'
+    category             = 'news, politics, religion, Vatican'
+    no_stylesheets       = True
+    INDEX                = 'http://www.vatican.va'
+    FEEDPAGE             = INDEX + '/news_services/or/or_quo/index.html'
+    CONTENTPAGE          = INDEX + '/news_services/or/or_quo/text.html'
+    use_embedded_content = False
+    encoding             = 'cp1252'
+    language             = 'it'
+    publication_type     = 'newspaper'
+
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+    def parse_index(self):
+        articles = []
+        articles.append({
+                          'title'      :self.title
+                         ,'date'       :''
+                         ,'url'        :self.CONTENTPAGE
+                         ,'description':''
+                        })
+        return [(self.title, articles)]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+

resources/recipes/lrb.recipe

@@ -1,6 +1,6 @@
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 '''
 lrb.co.uk
 '''
@@ -8,17 +8,20 @@ lrb.co.uk
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class LondonReviewOfBooks(BasicNewsRecipe):
-    title                 = u'London Review of Books'
-    __author__            = u'Darko Miletic'
-    description           = u'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
-    category              = 'news, literature, England'
-    publisher             = 'London Review of Books'
-    oldest_article        = 7
+    title                 = 'London Review of Books (free)'
+    __author__            = 'Darko Miletic'
+    description           = 'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB ltd.'
+    oldest_article        = 15
     max_articles_per_feed = 100
     language              = 'en_GB'
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
+    publication_type      = 'magazine'
+    masthead_url          = 'http://www.lrb.co.uk/assets/images/lrb_logo_big.gif'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
 
     conversion_options = {
                              'comments'  : description
@@ -27,13 +30,16 @@ class LondonReviewOfBooks(BasicNewsRecipe):
                             ,'publisher' : publisher
                          }
 
-    keep_only_tags = [dict(name='div' , attrs={'id'   :'main'})]
-    remove_tags = [
-                    dict(name='div' , attrs={'class':['pagetools','issue-nav-controls','nocss']})
-                   ,dict(name='div' , attrs={'id'   :['mainmenu','precontent','otherarticles']     })
-                   ,dict(name='span', attrs={'class':['inlineright','article-icons']})
-                   ,dict(name='ul'  , attrs={'class':'article-controls'})
-                   ,dict(name='p'   , attrs={'class':'meta-info'       })
-                  ]
+    keep_only_tags = [dict(attrs={'class':['article-body indent','letters','article-list']})]
+    remove_attributes = ['width','height']
 
     feeds = [(u'London Review of Books', u'http://www.lrb.co.uk/lrbrss.xml')]
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup('http://www.lrb.co.uk/')
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        if cover_item:
+           cover_url = 'http://www.lrb.co.uk' + cover_item.a.img['src']
+        return cover_url
+

resources/recipes/lrb_payed.recipe

@@ -0,0 +1,75 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+lrb.co.uk
+'''
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LondonReviewOfBooksPayed(BasicNewsRecipe):
+    title                 = 'London Review of Books'
+    __author__            = 'Darko Miletic'
+    description           = 'Subscription content. Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers'
+    category              = 'news, literature, UK'
+    publisher             = 'LRB Ltd.'
+    max_articles_per_feed = 100
+    language              = 'en_GB'
+    no_stylesheets        = True
+    delay                 = 1
+    use_embedded_content  = False
+    encoding              = 'utf-8'
+    INDEX                 = 'http://www.lrb.co.uk'
+    LOGIN                 = INDEX + '/login'
+    masthead_url          = INDEX + '/assets/images/lrb_logo_big.gif'
+    needs_subscription    = True
+    publication_type      = 'magazine'
+    extra_css             = ' body{font-family: Georgia,Palatino,"Palatino Linotype",serif} '
+
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.LOGIN)
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('p',attrs={'class':'cover'})
+        lrbtitle = self.title
+        if  cover_item:
+            self.cover_url = self.INDEX + cover_item.a.img['src']
+            content = self.INDEX + cover_item.a['href']
+            soup2 = self.index_to_soup(content)
+            sitem = soup2.find(attrs={'class':'article-list'})
+            lrbtitle = soup2.head.title.string
+            for item in sitem.findAll('a',attrs={'class':'title'}):
+                description = u''
+                title_prefix = u''
+                feed_link = item
+                if feed_link.has_key('href'):
+                    url   = self.INDEX + feed_link['href']
+                    title = title_prefix + self.tag_to_string(feed_link)
+                    date  = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+        return [(lrbtitle, articles)]
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    keep_only_tags = [dict(name='div' , attrs={'class':['article-body indent','letters']})]
+    remove_attributes = ['width','height']

resources/recipes/national_post.recipe

@@ -7,18 +7,18 @@ class NYTimes(BasicNewsRecipe):
     __author__  = 'Krittika Goyal'
     description = 'Canadian national newspaper'
     timefmt = ' [%d %b, %Y]'
-    needs_subscription = False
     language = 'en_CA'
+    needs_subscription = False
 
     no_stylesheets = True
     #remove_tags_before = dict(name='h1', attrs={'class':'heading'})
-    #remove_tags_after  = dict(name='td', attrs={'class':'newptool1'})
+    remove_tags_after  = dict(name='div', attrs={'class':'npStoryTools npWidth1-6 npRight npTxtStrong'})
     remove_tags = [
        dict(name='iframe'),
-       dict(name='div', attrs={'class':'story-tools'}),
+       dict(name='div', attrs={'class':['story-tools', 'npStoryTools npWidth1-6 npRight npTxtStrong']}),
        #dict(name='div', attrs={'id':['qrformdiv', 'inSection', 'alpha-inner']}),
        #dict(name='form', attrs={'onsubmit':''}),
-       #dict(name='table', attrs={'cellspacing':'0'}),
+       dict(name='ul', attrs={'class':'npTxtAlt npGroup npTxtCentre npStoryShare npTxtStrong npTxtDim'}),
     ]
 
    # def preprocess_html(self, soup):
@@ -37,7 +37,7 @@ class NYTimes(BasicNewsRecipe):
     def parse_index(self):
             soup = self.nejm_get_index()
 
-            div = soup.find(id='LegoText4')
+            div = soup.find(id='npContentMain')
 
             current_section = None
             current_articles = []
@@ -50,7 +50,7 @@ class NYTimes(BasicNewsRecipe):
                     current_section = self.tag_to_string(x)
                     current_articles = []
                     self.log('\tFound section:', current_section)
-                if current_section is not None and x.name == 'h3':
+                if current_section is not None and x.name == 'h5':
                     # Article found
                     title = self.tag_to_string(x)
                     a = x.find('a', href=lambda x: x and 'story' in x)
@@ -59,7 +59,7 @@ class NYTimes(BasicNewsRecipe):
                     url = a.get('href', False)
                     if not url or not title:
                         continue
-                    if url.startswith('story'):
+                    #if url.startswith('story'):
                     url = 'http://www.nationalpost.com/todays-paper/'+url
                     self.log('\t\tFound article:', title)
                     self.log('\t\t\t', url)
@@ -70,28 +70,11 @@ class NYTimes(BasicNewsRecipe):
                 feeds.append((current_section, current_articles))
 
             return feeds
-
     def preprocess_html(self, soup):
-        story = soup.find(name='div', attrs={'class':'triline'})
-        page2_link = soup.find('p','pagenav')
-        if page2_link:
-            atag = page2_link.find('a',href=True)
-            if atag:
-                page2_url = atag['href']
-                if page2_url.startswith('story'):
-                         page2_url = 'http://www.nationalpost.com/todays-paper/'+page2_url
-                elif page2_url.startswith( '/todays-paper/story.html'):
-                    page2_url = 'http://www.nationalpost.com/'+page2_url
-                page2_soup = self.index_to_soup(page2_url)
-                if page2_soup:
-                    page2_content = page2_soup.find('div','story-content')
-                    if page2_content:
-                        full_story = BeautifulSoup('<div></div>')
-                        full_story.insert(0,story)
-                        full_story.insert(1,page2_content)
-                        story = full_story
+        story = soup.find(name='div', attrs={'id':'npContentMain'})
+        ##td = heading.findParent(name='td')
+        ##td.extract()
         soup = BeautifulSoup('<html><head><title>t</title></head><body></body></html>')
         body = soup.find(name='body')
         body.insert(0, story)
         return soup
-

resources/recipes/new_scientist.recipe

@@ -32,15 +32,16 @@ class NewScientist(BasicNewsRecipe):
                         }
     preprocess_regexps = [(re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')]
 
-    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','nsblgposts','hldgalcols']})]
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
-                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools']})
+                    ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
                     ,dict(name='meta' , attrs={'name' :'description'                       })
+                    ,dict(name='a'    , attrs={'rel'  :'tag'                                })
                   ]
-    remove_tags_after = dict(attrs={'class':'nbpcopy'})
+    remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
     remove_attributes = ['height','width']
 
     feeds          = [

resources/recipes/nytimes.recipe

@@ -17,7 +17,7 @@ class NYTimes(BasicNewsRecipe):
     title       = 'New York Times Top Stories'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
     description = 'Top Stories from the New York Times'
 
     # List of sections typically included in Top Stories.  Use a keyword from the
@@ -79,6 +79,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -110,6 +111,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',

resources/recipes/nytimes_sub.recipe

@@ -13,14 +13,14 @@ Story
 import re, string, time
 from calibre import strftime
 from calibre.web.feeds.recipes import BasicNewsRecipe
-from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, NavigableString, Tag
+from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup, NavigableString, Tag
 
 class NYTimes(BasicNewsRecipe):
 
     title       = 'The New York Times'
     __author__  = 'GRiker'
     language = 'en'
-    requires_version = (0, 7, 3)
+    requires_version = (0, 7, 5)
 
     description = 'Daily news from the New York Times (subscription version)'
     allSectionKeywords = ['The Front Page', 'International','National','Obituaries','Editorials',
@@ -66,6 +66,7 @@ class NYTimes(BasicNewsRecipe):
                             'doubleRule',
                             'dottedLine',
                             'entry-meta',
+                            'entry-response module',
                             'icon enlargeThis',
                             'leftNavTabs',
                             'module box nav',
@@ -97,6 +98,7 @@ class NYTimes(BasicNewsRecipe):
                             'navigation',
                             'portfolioInline',
                             'relatedArticles',
+                            'respond',
                             'side_search',
                             'side_index',
                             'side_tool',
@@ -417,12 +419,11 @@ class NYTimes(BasicNewsRecipe):
 
         return soup
 
-    def postprocess_book(self, oeb, opts, log) :
-        print "\npostprocess_book()\n"
-
-        def extract_byline(href) :
-            # <meta name="byline" content=
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
+    def populate_article_metadata(self,article,soup,first):
+        '''
+        Extract author and description from article, add to article metadata
+        '''
+        def extract_author(soup):
             byline = soup.find('meta',attrs={'name':['byl','CLMST']})
             if byline :
                 author = byline['content']
@@ -432,50 +433,32 @@ class NYTimes(BasicNewsRecipe):
                 if byline:
                     author = byline.renderContents()
                 else:
-                    print "couldn't find byline in %s" % href
                     print soup.prettify()
                     return None
-            # Kill commas - Kindle switches to '&'
-            return re.sub(',','',author)
+            return author
 
-        def extract_description(href) :
-            soup = BeautifulSoup(str(oeb.manifest.hrefs[href]))
+        def extract_description(soup):
             description = soup.find('meta',attrs={'name':['description','description ']})
             if description :
-#                print repr(description['content'])
-#                print self.massageNCXText(description['content'])
                 return self.massageNCXText(description['content'])
             else:
                 # Take first paragraph of article
-                articleBody = soup.find('div',attrs={'id':'articleBody'})
-                if not articleBody:
+                articlebody = soup.find('div',attrs={'id':'articlebody'})
+                if not articlebody:
                     # Try again with class instead of id
-                    articleBody = soup.find('div',attrs={'class':'articleBody'})
-                    if not articleBody:
-                        print 'postprocess_book.extract_description(): Did not find <div id="articleBody">:'
+                    articlebody = soup.find('div',attrs={'class':'articlebody'})
+                    if not articlebody:
+                        print 'postprocess_book.extract_description(): Did not find <div id="articlebody">:'
                         print soup.prettify()
                         return None
-                paras = articleBody.findAll('p')
+                paras = articlebody.findAll('p')
                 for p in paras:
                     if p.renderContents() > '' :
                         return self.massageNCXText(self.tag_to_string(p,use_alt=False))
                 return None
 
-        # Method entry point here
-        # Single section toc looks different than multi-section tocs
-        if oeb.toc.depth() == 2 :
-            for article in oeb.toc :
-                if article.author is None :
-                    article.author = extract_byline(article.href)
-                if article.description is None :
-                    article.description = extract_description(article.href).decode('utf-8')
-        elif oeb.toc.depth() == 3 :
-            for section in oeb.toc :
-                for article in section :
-                    if article.author is None :
-                        article.author = extract_byline(article.href)
-                    if article.description is None :
-                        article.description = extract_description(article.href)
+        article.author = extract_author(soup)
+        article.summary = article.text_summary = extract_description(soup)
 
     def strip_anchors(self,soup):
         paras = soup.findAll(True)

resources/recipes/people_daily.recipe

@@ -0,0 +1,57 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1277129332(BasicNewsRecipe):
+    title          = u'People Daily - China'
+    oldest_article = 2
+    max_articles_per_feed = 100
+    __author__            = 'rty'
+
+    pubisher  = 'people.com.cn'
+    description           = 'People Daily Newspaper'
+    language = 'zh'
+    category              = 'News, China'
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    encoding               = 'GB2312'
+    conversion_options = {'linearize_tables':True}
+
+    feeds          = [(u'\u56fd\u5185\u65b0\u95fb', u'http://www.people.com.cn/rss/politics.xml'),
+       (u'\u56fd\u9645\u65b0\u95fb', u'http://www.people.com.cn/rss/world.xml'),
+       (u'\u7ecf\u6d4e\u65b0\u95fb', u'http://www.people.com.cn/rss/finance.xml'),
+       (u'\u4f53\u80b2\u65b0\u95fb', u'http://www.people.com.cn/rss/sports.xml'),
+       (u'\u53f0\u6e7e\u65b0\u95fb', u'http://www.people.com.cn/rss/haixia.xml')]
+    keep_only_tags = [
+                              dict(name='div', attrs={'class':'left_content'}),
+                               ]
+    remove_tags = [
+                    dict(name='table', attrs={'class':'title'}),
+                         ]
+    remove_tags_after = [
+                  dict(name='table', attrs={'class':'bianji'}),
+                         ]
+
+    def append_page(self, soup, appendtag, position):
+        pager = soup.find('img',attrs={'src':'/img/next_b.gif'})
+        if pager:
+           nexturl = self.INDEX + pager.a['href']
+           soup2 = self.index_to_soup(nexturl)
+           texttag = soup2.find('div', attrs={'class':'left_content'})
+           #for it in texttag.findAll(style=True):
+           #   del it['style']
+           newpos = len(texttag.contents)
+           self.append_page(soup2,texttag,newpos)
+           texttag.extract()
+           appendtag.insert(position,texttag)
+
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="content-type" content="text/html;charset=GB2312" />\n<meta http-equiv="content-language" content="utf-8" />'
+        soup.head.insert(0,mtag)
+        for item in soup.findAll(style=True):
+            del item['form']
+        self.append_page(soup, soup.body, 3)
+        #pager = soup.find('a',attrs={'class':'ab12'})
+        #if pager:
+        #   pager.extract()
+        return soup

src/calibre/customize/profiles.py

@@ -36,7 +36,7 @@ class Plugin(_Plugin):
         self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
         self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
 
-
+# Input profiles {{{
 class InputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -218,6 +218,8 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input,
 
 input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
 
+# }}}
+
 class OutputProfile(Plugin):
 
     author = 'Kovid Goyal'
@@ -237,11 +239,12 @@ class OutputProfile(Plugin):
     # If True the MOBI renderer on the device supports MOBI indexing
     supports_mobi_indexing = False
 
-    # Device supports displaying a nested TOC
-    supports_nested_toc = True
-
     # If True output should be optimized for a touchscreen interface
     touchscreen = False
+    touchscreen_news_css = ''
+    # A list of extra (beyond CSS 2.1) modules supported by the device
+    # Format is a cssutils profile dictionary (see iPad for example)
+    extra_css_modules = []
 
     @classmethod
     def tags_to_string(cls, tags):
@@ -256,8 +259,94 @@ class iPadOutput(OutputProfile):
     screen_size = (768, 1024)
     comic_screen_size = (768, 1024)
     dpi = 132.0
-    supports_nested_toc = False
+    extra_css_modules = [
+        {
+            'name':'webkit',
+            'props': { '-webkit-border-bottom-left-radius':'{length}',
+                '-webkit-border-bottom-right-radius':'{length}',
+                '-webkit-border-top-left-radius':'{length}',
+                '-webkit-border-top-right-radius':'{length}',
+                '-webkit-border-radius': r'{border-width}(\s+{border-width}){0,3}|inherit',
+            },
+            'macros': {'border-width': '{length}|medium|thick|thin'}
+        }
+    ]
     touchscreen = True
+    # touchscreen_news_css {{{
+    touchscreen_news_css = u'''
+			/* hr used in articles */
+            .caption_divider {
+            	border:#ccc 1px solid;
+				}
+
+            .touchscreen_navbar {
+                background:#ccc;
+                border:#ccc 1px solid;
+                border-collapse:separate;
+                border-spacing:1px;
+                margin-left: 5%;
+                margin-right: 5%;
+                width: 90%;
+                -webkit-border-radius:4px;
+                }
+            .touchscreen_navbar td {
+                background:#fff;
+                font-family:Helvetica;
+                font-size:90%;
+                padding: 5px;
+                text-align:center;
+                }
+            .touchscreen_navbar td:first-child {
+				-webkit-border-top-left-radius:4px;
+				-webkit-border-bottom-left-radius:4px;
+				}
+			.touchscreen_navbar td:last-child {
+				-webkit-border-top-right-radius:4px;
+				-webkit-border-bottom-right-radius:4px;
+				}
+
+            .feed_link {
+                font-style: italic;
+                }
+
+			/* Index formatting */
+			.publish_date {
+				text-align:center;
+				}
+			.divider {
+				border-bottom:1em solid white;
+				border-top:1px solid gray;
+				}
+
+            /* Feed summary formatting */
+            .feed_title {
+                text-align: center;
+                font-size: 160%;
+                }
+
+            .summary_headline {
+                font-weight:bold;
+                text-align:left;
+				}
+
+            .summary_byline {
+                text-align:left;
+                font-family:monospace;
+				}
+
+            .summary_text {
+                text-align:left;
+				}
+
+            .feed {
+                font-family:sans-serif;
+                font-weight:bold;
+                font-size:larger;
+				}
+
+        '''
+        # }}}
+
 
 class SonyReaderOutput(OutputProfile):
 

src/calibre/customize/ui.py

@@ -151,13 +151,13 @@ def reread_filetype_plugins():
 
 
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
-    occasion = {'import':_on_import, 'preprocess':_on_preprocess,
+    occasion_plugins = {'import':_on_import, 'preprocess':_on_preprocess,
                 'postprocess':_on_postprocess}[occasion]
     customization = config['plugin_customization']
     if ft is None:
         ft = os.path.splitext(path_to_file)[-1].lower().replace('.', '')
     nfp = path_to_file
-    for plugin in occasion.get(ft, []):
+    for plugin in occasion_plugins.get(ft, []):
         if is_disabled(plugin):
             continue
         plugin.site_customization = customization.get(plugin.name, '')

src/calibre/devices/android/driver.py

@@ -45,8 +45,8 @@ class ANDROID(USBMS):
             'GT-I5700', 'SAMSUNG']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD',
-            'PROD_GT-I9000']
-    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PROD_GT-I9000_CARD']
+            'PR OD_GT-I9000']
+    WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'PR OD_GT-I9000_CARD']
 
     OSX_MAIN_MEM = 'HTC Android Phone Media'
 

src/calibre/devices/apple/driver.py

@@ -10,13 +10,15 @@ from calibre.constants import __appname__, __version__, DEBUG
 from calibre import fit_image
 from calibre.constants import isosx, iswindows
 from calibre.devices.errors import UserFeedback
+from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre.devices.interface import DevicePlugin
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.epub import set_metadata
 from calibre.library.server.utils import strftime
-from calibre.utils.config import Config, config_dir
+from calibre.utils.config import config_dir
 from calibre.utils.date import isoformat, now, parse_date
+from calibre.utils.localization import get_lang
 from calibre.utils.logging import Log
 from calibre.utils.zipfile import ZipFile
 
@@ -33,8 +35,15 @@ if isosx:
 if iswindows:
     import pythoncom, win32com.client
 
+class DriverBase(DeviceConfig, DevicePlugin):
+    # Needed for config_widget to work
+    FORMATS = ['epub', 'pdf']
 
-class ITUNES(DevicePlugin):
+    @classmethod
+    def _config_base_name(cls):
+        return 'iTunes'
+
+class ITUNES(DriverBase):
     '''
     Calling sequences:
     Initialization:
@@ -78,18 +87,22 @@ class ITUNES(DevicePlugin):
     supported_platforms = ['osx','windows']
     author = 'GRiker'
     #: The version of this plugin as a 3-tuple (major, minor, revision)
-    version        = (0,7,0)
+    version        = (0,8,0)
 
     OPEN_FEEDBACK_MESSAGE = _(
         'Apple device detected, launching iTunes, please wait ...')
 
-    FORMATS = ['epub']
 
     # Product IDs:
-    #  0x1292:iPhone 3G
-    #  0x129a:iPad
+    #  0x1291   iPod Touch
+    #  0x1292   iPhone 3G
+    #  0x1293   iPod Touch 2G
+    #  0x1294   iPhone 3GS
+    #  0x1297   iPhone 4
+    #  0x1299   iPod Touch 3G
+    #  0x129a   iPad
     VENDOR_ID = [0x05ac]
-    PRODUCT_ID = [0x129a]
+    PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a]
     BCD = [0x01]
 
     # iTunes enumerations
@@ -141,6 +154,10 @@ class ITUNES(DevicePlugin):
                 'SongNames',
                 ]
 
+    # Cover art size limits
+    MAX_COVER_WIDTH = 510
+    MAX_COVER_HEIGHT = 680
+
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
@@ -159,7 +176,6 @@ class ITUNES(DevicePlugin):
     sources = None
     update_msg = None
     update_needed = False
-    use_series_data = True
 
     # Public methods
     def add_books_to_metadata(self, locations, metadata, booklists):
@@ -173,16 +189,17 @@ class ITUNES(DevicePlugin):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+        if DEBUG:
+            self.log.info("ITUNES.add_books_to_metadata()")
 
         task_count = float(len(self.update_list))
 
         # Delete any obsolete copies of the book from the booklist
         if self.update_list:
-            if True:
-                self.log.info("ITUNES.add_books_to_metadata()")
-                #self._dump_booklist(booklists[0], header='before',indent=2)
-                #self._dump_update_list(header='before',indent=2)
-                #self._dump_cached_books(header='before',indent=2)
+            if False:
+                self._dump_booklist(booklists[0], header='before',indent=2)
+                self._dump_update_list(header='before',indent=2)
+                self._dump_cached_books(header='before',indent=2)
 
             for (j,p_book) in enumerate(self.update_list):
                 if False:
@@ -230,12 +247,12 @@ class ITUNES(DevicePlugin):
 
         # Add new books to booklists[0]
         for new_book in locations[0]:
-            if False:
+            if DEBUG:
                 self.log.info("  adding '%s' by '%s' to booklists[0]" %
                     (new_book.title, new_book.author))
             booklists[0].append(new_book)
 
-        if False:
+        if DEBUG:
             self._dump_booklist(booklists[0],header='after',indent=2)
             self._dump_cached_books(header='after',indent=2)
 
@@ -329,7 +346,8 @@ class ITUNES(DevicePlugin):
                              'title':book.Name,
                              'author':book.Artist,
                              'lib_book':library_books[this_book.path] if this_book.path in library_books else None,
-                             'uuid': book.Composer
+                             'uuid': book.Composer,
+                             'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub'
                              }
 
                             if self.report_progress is not None:
@@ -343,9 +361,9 @@ class ITUNES(DevicePlugin):
                 if self.report_progress is not None:
                     self.report_progress(1.0, _('finished'))
                 self.cached_books = cached_books
-#                 if DEBUG:
-#                     self._dump_booklist(booklist, 'returning from books():')
-#                     self._dump_cached_books('returning from books():')
+                if DEBUG:
+                    self._dump_booklist(booklist, 'returning from books()', indent=2)
+                    self._dump_cached_books('returning from books()',indent=2)
                 return booklist
         else:
             return []
@@ -506,6 +524,19 @@ class ITUNES(DevicePlugin):
         '''
         return (None,None)
 
+    @classmethod
+    def config_widget(cls):
+        '''
+        Return a QWidget with settings for the device interface
+        '''
+        cw = DriverBase.config_widget()
+        # Turn off the Save template
+        cw.opt_save_template.setVisible(False)
+        cw.label.setVisible(False)
+        # Repurpose the checkbox
+        cw.opt_read_metadata.setText(_("Use Series as Category in iTunes/iBooks"))
+        return cw
+
     def delete_books(self, paths, end_session=True):
         '''
         Delete books at paths on device.
@@ -685,6 +716,9 @@ class ITUNES(DevicePlugin):
         @param booklists:  A tuple containing the result of calls to
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
+
+        NB: This will not find books that were added by a different installation of calibre
+            as uuids are different
         '''
         if DEBUG:
             self.log.info("ITUNES.remove_books_from_metadata()")
@@ -732,17 +766,6 @@ class ITUNES(DevicePlugin):
         '''
         self.report_progress = report_progress
 
-    def settings(self):
-        '''
-        Should return an opts object. The opts object should have one attribute
-        `format_map` which is an ordered list of formats for the device.
-        '''
-        klass = self if isinstance(self, type) else self.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
-        c.add_opt('format_map', default=self.FORMATS,
-            help=_('Ordered list of formats the device will accept'))
-        return c.parse()
-
     def sync_booklists(self, booklists, end_session=True):
         '''
         Update metadata on device.
@@ -750,6 +773,10 @@ class ITUNES(DevicePlugin):
                                 (L{books}(oncard=None), L{books}(oncard='carda'),
                                 L{books}(oncard='cardb')).
         '''
+
+        if DEBUG:
+            self.log.info("ITUNES.sync_booklists()")
+
         if self.update_needed:
             if DEBUG:
                 self.log.info(' calling _update_device')
@@ -812,29 +839,33 @@ class ITUNES(DevicePlugin):
         self.problem_msg = _("Some cover art could not be converted.\n"
                                      "Click 'Show Details' for a list.")
 
-        if False:
+        if DEBUG:
             self.log.info("ITUNES.upload_books()")
             self._dump_files(files, header='upload_books()',indent=2)
             self._dump_update_list(header='upload_books()',indent=2)
+            #self.log.info("  self.settings().format_map: %s" % self.settings().format_map)
 
         if isosx:
             for (i,file) in enumerate(files):
+                format = file.rpartition('.')[2].lower()
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
                 self._remove_existing_copy(path, metadata[i])
-                fpath = self._get_fpath(file, metadata[i], update_md=True)
+                fpath = self._get_fpath(file, metadata[i], format, update_md=True)
                 db_added, lb_added = self._add_new_copy(fpath, metadata[i])
-                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added)
-                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
+                this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                 new_booklist.append(this_book)
                 self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                 # Add new_book to self.cached_paths
                 self.cached_books[this_book.path] = {
-                    'title': metadata[i].title,
                    'author': metadata[i].author,
-                 'lib_book': lb_added,
                  'dev_book': db_added,
-                     'uuid': metadata[i].uuid}
+                   'format': format,
+                 'lib_book': lb_added,
+                    'title': metadata[i].title,
+                     'uuid': metadata[i].uuid }
+
 
                 # Report progress
                 if self.report_progress is not None:
@@ -846,9 +877,10 @@ class ITUNES(DevicePlugin):
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
 
                 for (i,file) in enumerate(files):
+                    format = file.rpartition('.')[2].lower()
                     path = self.path_template % (metadata[i].title, metadata[i].author[0])
                     self._remove_existing_copy(path, metadata[i])
-                    fpath = self._get_fpath(file, metadata[i], update_md=True)
+                    fpath = self._get_fpath(file, metadata[i],format, update_md=True)
                     db_added, lb_added = self._add_new_copy(fpath, metadata[i])
 
                     if self.manual_sync_mode and not db_added:
@@ -857,17 +889,18 @@ class ITUNES(DevicePlugin):
                                             "Click 'Show Details...' for affected books.")
                         self.problem_titles.append("'%s' by %s" % (metadata[i].title, metadata[i].author[0]))
 
-                    thumb = self._cover_to_thumb(path, metadata[i], lb_added, db_added)
-                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb)
+                    thumb = self._cover_to_thumb(path, metadata[i], db_added, lb_added, format)
+                    this_book = self._create_new_book(fpath, metadata[i], path, db_added, lb_added, thumb, format)
                     new_booklist.append(this_book)
                     self._update_iTunes_metadata(metadata[i], db_added, lb_added, this_book)
 
                     # Add new_book to self.cached_paths
                     self.cached_books[this_book.path] = {
-                        'title': metadata[i].title,
                        'author': metadata[i].author[0],
-                     'lib_book': lb_added,
                      'dev_book': db_added,
+                       'format': format,
+                     'lib_book': lb_added,
+                        'title': metadata[i].title,
                          'uuid': metadata[i].uuid}
 
                     # Report progress
@@ -968,7 +1001,8 @@ class ITUNES(DevicePlugin):
                         db_added = self._find_device_book(
                                      {'title': metadata.title,
                                      'author': metadata.authors[0],
-                                       'uuid': metadata.uuid})
+                                       'uuid': metadata.uuid,
+                                     'format': fpath.rpartition('.')[2].lower()})
 
                     return db_added
 
@@ -1021,7 +1055,8 @@ class ITUNES(DevicePlugin):
                 added = self._find_library_book(
                     { 'title': metadata.title,
                      'author': metadata.author[0],
-                       'uuid': metadata.uuid})
+                       'uuid': metadata.uuid,
+                     'format': file.rpartition('.')[2].lower()})
         return added
 
     def _add_new_copy(self, fpath, metadata):
@@ -1047,23 +1082,50 @@ class ITUNES(DevicePlugin):
 
         return db_added, lb_added
 
-    def _cover_to_thumb(self, path, metadata, db_added, lb_added):
+    def _cover_to_thumb(self, path, metadata, db_added, lb_added, format):
         '''
         assumes pythoncom wrapper for db_added
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
         self.log.info(" ITUNES._cover_to_thumb()")
+
         thumb = None
         if metadata.cover:
+
+            if (format == 'epub'):
+                # Pre-shrink cover
+                # self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT
+                try:
+                    img = PILImage.open(metadata.cover)
+                    width = img.size[0]
+                    height = img.size[1]
+                    scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT)
+                    if scaled:
+                        if DEBUG:
+                            self.log.info("   '%s' scaled from %sx%s to %sx%s" %
+                                          (metadata.cover,width,height,nwidth,nheight))
+                        img = img.resize((nwidth, nheight), PILImage.ANTIALIAS)
+                        cd = cStringIO.StringIO()
+                        img.convert('RGB').save(cd, 'JPEG')
+                        cover_data = cd.getvalue()
+                        cd.close()
+                    else:
+                        with open(metadata.cover,'r+b') as cd:
+                            cover_data = cd.read()
+                except:
+                    self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
+                    self.log.error("  error scaling '%s' for '%s'" % (metadata.cover,metadata.title))
+                    return thumb
+
                 if isosx:
-                cover_data = open(metadata.cover,'rb')
                     if lb_added:
-                    lb_added.artworks[1].data_.set(cover_data.read())
+                        lb_added.artworks[1].data_.set(cover_data)
 
                     if db_added:
                         # The following command generates an error, but the artwork does in fact
                         # get sent to the device.  Seems like a bug in Apple's automation interface
                         try:
-                        db_added.artworks[1].data_.set(cover_data.read())
+                            db_added.artworks[1].data_.set(cover_data)
                         except:
                             if DEBUG:
                                 self.log.warning("  iTunes automation interface reported an error"
@@ -1076,17 +1138,26 @@ class ITUNES(DevicePlugin):
 
 
                 elif iswindows:
+                    # Write the data to a real file for Windows iTunes
+                    tc = os.path.join(tempfile.gettempdir(), "cover.jpg")
+                    with open(tc,'wb') as tmp_cover:
+                        tmp_cover.write(cover_data)
+
                     if lb_added:
                         if lb_added.Artwork.Count:
-                        lb_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
+                            lb_added.Artwork.Item(1).SetArtworkFromFile(tc)
                         else:
-                        lb_added.AddArtworkFromFile(metadata.cover)
+                            lb_added.AddArtworkFromFile(tc)
 
                     if db_added:
                         if db_added.Artwork.Count:
-                        db_added.Artwork.Item(1).SetArtworkFromFile(metadata.cover)
+                            db_added.Artwork.Item(1).SetArtworkFromFile(tc)
                         else:
-                        db_added.AddArtworkFromFile(metadata.cover)
+                            db_added.AddArtworkFromFile(tc)
+
+            elif format == 'pdf':
+                if DEBUG:
+                    self.log.info("   unable to set PDF cover via automation interface")
 
             try:
                 # Resize for thumb
@@ -1097,6 +1168,7 @@ class ITUNES(DevicePlugin):
                 of = cStringIO.StringIO()
                 im.convert('RGB').save(of, 'JPEG')
                 thumb = of.getvalue()
+                of.close()
 
                 # Refresh the thumbnail cache
                 if DEBUG:
@@ -1105,14 +1177,15 @@ class ITUNES(DevicePlugin):
                 zfw = ZipFile(archive_path, mode='a')
                 thumb_path = path.rpartition('.')[0] + '.jpg'
                 zfw.writestr(thumb_path, thumb)
-                zfw.close()
             except:
                 self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0]))
                 self.log.error("  error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title))
+            finally:
+                zfw.close()
 
         return thumb
 
-    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb):
+    def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb, format):
         '''
         '''
         if DEBUG:
@@ -1122,6 +1195,7 @@ class ITUNES(DevicePlugin):
 
         this_book.db_id = None
         this_book.device_collections = []
+        this_book.format = format
         this_book.library_id = lb_added
         this_book.path = path
         this_book.thumbnail = thumb
@@ -1134,13 +1208,13 @@ class ITUNES(DevicePlugin):
                 try:
                     this_book.datetime = parse_date(str(lb_added.date_added())).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
             elif db_added:
                 this_book.size = self._get_device_book_size(fpath, db_added.size())
                 try:
                     this_book.datetime = parse_date(str(db_added.date_added())).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
 
         elif iswindows:
             if lb_added:
@@ -1148,13 +1222,13 @@ class ITUNES(DevicePlugin):
                 try:
                     this_book.datetime = parse_date(str(lb_added.DateAdded)).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
             elif db_added:
                 this_book.size = self._get_device_book_size(fpath, db_added.Size)
                 try:
                     this_book.datetime = parse_date(str(db_added.DateAdded)).timetuple()
                 except:
-                    pass
+                    this_book.datetime = time.gmtime()
 
         return this_book
 
@@ -1319,10 +1393,11 @@ class ITUNES(DevicePlugin):
                   self.cached_books[cb]['uuid']))
         elif iswindows:
             for cb in self.cached_books.keys():
-                self.log.info("%s%-40.40s %-30.30s %s" %
+                self.log.info("%s%-40.40s %-30.30s %-4.4s %s" %
                  (' '*indent,
                   self.cached_books[cb]['title'],
                   self.cached_books[cb]['author'],
+                  self.cached_books[cb]['format'],
                   self.cached_books[cb]['uuid']))
 
         self.log.info()
@@ -1338,8 +1413,9 @@ class ITUNES(DevicePlugin):
         fnames = zf.namelist()
         opf = [x for x in fnames if '.opf' in x][0]
         if opf:
-            opf_raw = cStringIO.StringIO(zf.read(opf)).getvalue()
-            soup = BeautifulSoup(opf_raw)
+            opf_raw = cStringIO.StringIO(zf.read(opf))
+            soup = BeautifulSoup(opf_raw.getvalue())
+            opf_raw.close()
             title = soup.find('dc:title').renderContents()
             author = soup.find('dc:creator').renderContents()
             ts = soup.find('meta',attrs={'name':'calibre:timestamp'})
@@ -1440,6 +1516,22 @@ class ITUNES(DevicePlugin):
                                 self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: '%s - %s'" % (title,author))
+                    hits = dev_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1509,6 +1601,22 @@ class ITUNES(DevicePlugin):
                                 self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
                             return hit
 
+                # PDF metadata was rewritten at export as 'safe(title) - safe(author)'
+                if search['format'] == 'pdf':
+                    title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title'])
+                    author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author'])
+                    if DEBUG:
+                        self.log.info("   searching by name: %s - %s" % (title,author))
+                    hits = lib_books.Search('%s - %s' % (title,author),
+                                             self.SearchField.index('All'))
+                    if hits:
+                        hit = hits[0]
+                        self.log.info("   found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer))
+                        return hit
+                    else:
+                        if DEBUG:
+                            self.log.info("   no PDF hits")
+
                 attempts -= 1
                 time.sleep(0.5)
                 if DEBUG:
@@ -1523,10 +1631,12 @@ class ITUNES(DevicePlugin):
         Convert iTunes artwork to thumbnail
         Cache generated thumbnails
         cache_dir = os.path.join(config_dir, 'caches', 'itunes')
+        as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
 
         archive_path = os.path.join(self.cache_dir, "thumbs.zip")
         thumb_path = book_path.rpartition('.')[0] + '.jpg'
+        format = book_path.rpartition('.')[2].lower()
 
         try:
             zfr = ZipFile(archive_path)
@@ -1539,22 +1649,30 @@ class ITUNES(DevicePlugin):
 
         self.log.info(" ITUNES._generate_thumbnail()")
         if isosx:
+            if format == 'epub':
                 try:
+                    if False:
+                        self.log.info("   fetching artwork from %s\n   %s" % (book_path,book))
                     # Resize the cover
                     data = book.artworks[1].raw_data().data
                     #self._dump_hex(data[:256])
-                im = PILImage.open(cStringIO.StringIO(data))
+                    img_data = cStringIO.StringIO(data)
+                    im = PILImage.open(img_data)
                     scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
                     im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
+                    img_data.close()
+
                     thumb = cStringIO.StringIO()
                     im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
+                    thumb.close()
 
                     # Cache the tagged thumb
                     if DEBUG:
                         self.log.info("  generated thumb for '%s', caching" % book.name())
-                zfw.writestr(thumb_path, thumb.getvalue())
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
-                return thumb.getvalue()
+                    return thumb_data
                 except:
                     self.log.error("  error generating thumb for '%s'" % book.name())
                     try:
@@ -1562,14 +1680,19 @@ class ITUNES(DevicePlugin):
                     except:
                         pass
                     return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
+                return None
 
         elif iswindows:
 
             if not book.Artwork.Count:
                 if DEBUG:
-                    self.log.info("  no artwork available")
+                    self.log.info("  no artwork available for '%s'" % book.Name)
                 return None
 
+            if format == 'epub':
                 # Save the cover from iTunes
                 try:
                     tmp_thumb = os.path.join(tempfile.gettempdir(), "thumb.%s" % self.ArtworkFormat[book.Artwork.Item(1).Format])
@@ -1580,14 +1703,16 @@ class ITUNES(DevicePlugin):
                     im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
                     thumb = cStringIO.StringIO()
                     im.convert('RGB').save(thumb,'JPEG')
+                    thumb_data = thumb.getvalue()
                     os.remove(tmp_thumb)
+                    thumb.close()
 
                     # Cache the tagged thumb
                     if DEBUG:
                         self.log.info("  generated thumb for '%s', caching" % book.Name)
-                zfw.writestr(thumb_path, thumb.getvalue())
+                    zfw.writestr(thumb_path, thumb_data)
                     zfw.close()
-                return thumb.getvalue()
+                    return thumb_data
                 except:
                     self.log.error("  error generating thumb for '%s'" % book.Name)
                     try:
@@ -1595,11 +1720,18 @@ class ITUNES(DevicePlugin):
                     except:
                         pass
                     return None
+            else:
+                if DEBUG:
+                    self.log.info("  unable to generate PDF thumbs")
+                return None
 
     def _get_device_book_size(self, file, compressed_size):
         '''
         Calculate the exploded size of file
         '''
+        exploded_file_size = compressed_size
+        format = file.rpartition('.')[2].lower()
+        if format == 'epub':
             myZip = ZipFile(file,'r')
             myZipList = myZip.infolist()
             exploded_file_size = 0
@@ -1701,7 +1833,7 @@ class ITUNES(DevicePlugin):
                         self.log.error("  no iPad|Books playlist found")
                 return pl
 
-    def _get_fpath(self,file, metadata, update_md=False):
+    def _get_fpath(self,file, metadata, format, update_md=False):
         '''
         If the database copy will be deleted after upload, we have to
         use file (the PersistentTemporaryFile), which will be around until
@@ -1725,7 +1857,7 @@ class ITUNES(DevicePlugin):
             if DEBUG:
                 self.log.info("   file will be deleted after upload")
 
-        if update_md:
+        if format == 'epub' and update_md:
             self._update_epub_metadata(fpath, metadata)
 
         return fpath
@@ -1775,10 +1907,14 @@ class ITUNES(DevicePlugin):
                             # Collect calibre orphans - remnants of recipe uploads
                             path = self.path_template % (book.name(), book.artist())
                             if str(book.description()).startswith(self.description_prefix):
+                                try:
                                     if book.location() == appscript.k.missing_value:
                                         library_orphans[path] = book
                                         if False:
                                             self.log.info("   found iTunes PTF '%s' in Library|Books" % book.name())
+                                except:
+                                    if DEBUG:
+                                        self.log.error("   iTunes returned an error returning .location() with %s" % book.name())
 
                             library_books[path] = book
                             if DEBUG:
@@ -1937,10 +2073,22 @@ class ITUNES(DevicePlugin):
                  (self.iTunes.name(), self.iTunes.version(), self.initial_status,
                   self.version[0],self.version[1],self.version[2]))
                 self.log.info("  iTunes_media: %s" % self.iTunes_media)
+
         if iswindows:
             '''
             Launch iTunes if not already running
             Assumes pythoncom wrapper
+
+            *** Current implementation doesn't handle UNC paths correctly,
+                and python has two incompatible methods to parse UNCs:
+                os.path.splitdrive() and os.path.splitunc()
+                need to use os.path.normpath on result of splitunc()
+
+                Once you have the //server/share, convert with os.path.normpath('//server/share')
+                os.path.splitdrive doesn't work as advertised, so use os.path.splitunc
+                os.path.splitunc("//server/share") returns ('//server/share','')
+                os.path.splitunc("C:/Documents") returns ('c:','/documents')
+                os.path.normpath("//server/share") returns "\\\\server\\share"
             '''
             # Instantiate iTunes
             self.iTunes = win32com.client.Dispatch("iTunes.Application")
@@ -1949,17 +2097,23 @@ class ITUNES(DevicePlugin):
             self.initial_status = 'launched'
 
             # Read the current storage path for iTunes media from the XML file
+            media_dir = ''
+            string = None
             with open(self.iTunes.LibraryXMLPath, 'r') as xml:
-                soup = BeautifulSoup(xml.read().decode('utf-8'))
-                mf = soup.find('key',text="Music Folder").parent
-                string = mf.findNext('string').renderContents()
+                for line in xml:
+                    if line.strip().startswith('<key>Music Folder'):
+                        soup = BeautifulSoup(line)
+                        string = soup.find('string').renderContents()
                         media_dir = os.path.abspath(string[len('file://localhost/'):].replace('%20',' '))
+                        break
                 if os.path.exists(media_dir):
                     self.iTunes_media = media_dir
-                else:
+                elif hasattr(string,'parent'):
                     self.log.error("  could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath)
                     self.log.error("  %s" % string.parent.prettify())
                     self.log.error("  '%s' not found" % media_dir)
+                else:
+                    self.log.error("  no media dir found: string: %s" % string)
 
             if DEBUG:
                 self.log.info("  %s %s" % (__appname__, __version__))
@@ -2028,7 +2182,9 @@ class ITUNES(DevicePlugin):
             # Delete existing from Library|Books, add to self.update_list
             # for deletion from booklist[0] during add_books_to_metadata
             for book in self.cached_books:
-                if self.cached_books[book]['uuid'] == metadata.uuid:
+                if (self.cached_books[book]['uuid'] == metadata.uuid) or \
+                   (self.cached_books[book]['title'] == metadata.title and \
+                    self.cached_books[book]['author'] == metadata.authors[0]):
                     self.update_list.append(self.cached_books[book])
                     self._remove_from_iTunes(self.cached_books[book])
                     if DEBUG:
@@ -2036,7 +2192,7 @@ class ITUNES(DevicePlugin):
                     break
                 else:
                     if DEBUG:
-                        self.log.info("  '%s' not in cached_books" % metadata.title)
+                        self.log.info("  '%s' not found in cached_books" % metadata.title)
 
     def _remove_from_device(self, cached_book):
         '''
@@ -2119,8 +2275,8 @@ class ITUNES(DevicePlugin):
                 path = book.Location
 
             if book:
+                if self.iTunes_media and path.startswith(self.iTunes_media):
                     storage_path = os.path.split(path)
-                if path.startswith(self.iTunes_media):
                     if DEBUG:
                         self.log.info("   removing '%s' at %s" %
                             (cached_book['title'], path))
@@ -2158,12 +2314,15 @@ class ITUNES(DevicePlugin):
             fnames = zf_opf.namelist()
             opf = [x for x in fnames if '.opf' in x][0]
             if opf:
-                opf_raw = cStringIO.StringIO(zf_opf.read(opf)).getvalue()
-                soup = BeautifulSoup(opf_raw)
+                opf_raw = cStringIO.StringIO(zf_opf.read(opf))
+                soup = BeautifulSoup(opf_raw.getvalue())
+                opf_raw.close()
+
+                # Touch existing calibre timestamp
                 md = soup.find('metadata')
+                if md:
                     ts = md.find('meta',attrs={'name':'calibre:timestamp'})
                     if ts:
-                    # Touch existing calibre timestamp
                         timestamp = ts['content']
                         old_ts = parse_date(timestamp)
                         metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour,
@@ -2172,6 +2331,17 @@ class ITUNES(DevicePlugin):
                         metadata.timestamp = isoformat(now())
                         if DEBUG:
                             self.log.info("   add timestamp: %s" % metadata.timestamp)
+                else:
+                    metadata.timestamp = isoformat(now())
+                    if DEBUG:
+                        self.log.warning("   missing <metadata> block in OPF file")
+                        self.log.info("   add timestamp: %s" % metadata.timestamp)
+
+                # Force the language declaration for iBooks 1.1
+                metadata.language = get_lang()
+                if DEBUG:
+                    self.log.info("   rewriting language: <dc:language>%s</dc:language>" % metadata.language)
+
             zf_opf.close()
 
             # If 'News' in tags, tweak the title/author for friendlier display in iBooks
@@ -2257,6 +2427,9 @@ class ITUNES(DevicePlugin):
                 lb_added.enabled.set(True)
                 lb_added.sort_artist.set(metadata.author_sort.title())
                 lb_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.artist.set(metadata.authors[0])
+                    lb_added.name.set(metadata.title)
 
             if db_added:
                 db_added.album.set(metadata.title)
@@ -2265,6 +2438,9 @@ class ITUNES(DevicePlugin):
                 db_added.enabled.set(True)
                 db_added.sort_artist.set(metadata.author_sort.title())
                 db_added.sort_name.set(this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.artist.set(metadata.authors[0])
+                    db_added.name.set(metadata.title)
 
             if metadata.comments:
                 if lb_added:
@@ -2284,7 +2460,9 @@ class ITUNES(DevicePlugin):
 
             # Set genre from series if available, else first alpha tag
             # Otherwise iTunes grabs the first dc:subject from the opf metadata
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.sort_name.set("%s %03d" % (metadata.series, metadata.series_index))
                     lb_added.genre.set(metadata.series)
@@ -2298,6 +2476,9 @@ class ITUNES(DevicePlugin):
                     db_added.episode_number.set(metadata.series_index)
 
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   %susing Tag as Genre" %
+                                  "no Series name available, " if self.settings().read_metadata else '')
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
@@ -2314,6 +2495,9 @@ class ITUNES(DevicePlugin):
                 lb_added.Enabled = True
                 lb_added.SortArtist = (metadata.author_sort.title())
                 lb_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    lb_added.Artist = metadata.authors[0]
+                    lb_added.Name = metadata.title
 
             if db_added:
                 db_added.Album = metadata.title
@@ -2322,6 +2506,9 @@ class ITUNES(DevicePlugin):
                 db_added.Enabled = True
                 db_added.SortArtist = (metadata.author_sort.title())
                 db_added.SortName = (this_book.title_sorter)
+                if this_book.format == 'pdf':
+                    db_added.Artist = metadata.authors[0]
+                    db_added.Name = metadata.title
 
             if metadata.comments:
                 if lb_added:
@@ -2345,7 +2532,9 @@ class ITUNES(DevicePlugin):
             # Otherwise iBooks uses first <dc:subject> from opf
             # iTunes balks on setting EpisodeNumber, but it sticks (9.1.1.12)
 
-            if self.use_series_data and metadata.series:
+            if metadata.series and self.settings().read_metadata:
+                if DEBUG:
+                    self.log.info("   using Series name as Genre")
                 if lb_added:
                     lb_added.SortName = "%s %03d" % (metadata.series, metadata.series_index)
                     lb_added.Genre = metadata.series
@@ -2365,6 +2554,8 @@ class ITUNES(DevicePlugin):
                             self.log.warning("  iTunes automation interface reported an error"
                                              " setting EpisodeNumber on iDevice")
             elif metadata.tags:
+                if DEBUG:
+                    self.log.info("   using Tag as Genre")
                 for tag in metadata.tags:
                     if self._is_alpha(tag[0]):
                         if lb_added:
@@ -2429,8 +2620,6 @@ class BookList(list):
 class Book(MetaInformation):
     '''
     A simple class describing a book in the iTunes Books Library.
-    Q's:
-    - Should thumbnail come from calibre if available?
     - See ebooks.metadata.__init__ for all fields
     '''
     def __init__(self,title,author):

src/calibre/devices/interface.py

@@ -59,7 +59,7 @@ class DevicePlugin(Plugin):
             return cls.__name__
         return cls.name
 
-
+    # Device detection {{{
     def test_bcd_windows(self, device_id, bcd):
         if bcd is None or len(bcd) == 0:
             return True
@@ -152,6 +152,7 @@ class DevicePlugin(Plugin):
                                     return True, dev
         return False, None
 
+    # }}}
 
     def reset(self, key='-1', log_packets=False, report_progress=None,
             detected_device=None) :
@@ -372,14 +373,12 @@ class DevicePlugin(Plugin):
     @classmethod
     def settings(cls):
         '''
-        Should return an opts object. The opts object should have one attribute
+        Should return an opts object. The opts object should have at least one attribute
         `format_map` which is an ordered list of formats for the device.
         '''
         raise NotImplementedError()
 
 
-
-
 class BookList(list):
     '''
     A list of books. Each Book object must have the fields:

src/calibre/devices/kindle/driver.py

@@ -429,6 +429,7 @@ class Bookmark():
                 entries, = unpack('>I', data[9:13])
                 current_entry = 0
                 e_base = 0x0d
+                self.pdf_page_offset = 0
                 while current_entry < entries:
                     '''
                     location, = unpack('>I', data[e_base+2:e_base+6])

src/calibre/devices/prs505/driver.py

@@ -99,7 +99,7 @@ class PRS505(USBMS):
         if self._card_b_prefix is not None:
             if not write_cache(self._card_b_prefix):
                 self._card_b_prefix = None
-
+        self.booklist_class.rebuild_collections = self.rebuild_collections
 
     def get_device_information(self, end_session=True):
         return (self.gui_name, '', '', '')
@@ -145,7 +145,7 @@ class PRS505(USBMS):
                 blists[i] = booklists[i]
         opts = self.settings()
         if opts.extra_customization:
-            collections = [x.strip() for x in
+            collections = [x.lower().strip() for x in
                     opts.extra_customization.split(',')]
         else:
             collections = []
@@ -156,4 +156,10 @@ class PRS505(USBMS):
         USBMS.sync_booklists(self, booklists, end_session=end_session)
         debug_print('PRS505: finished sync_booklists')
 
+    def rebuild_collections(self, booklist, oncard):
+        debug_print('PRS505: started rebuild_collections on card', oncard)
+        c = self.initialize_XML_cache()
+        c.rebuild_collections(booklist, {'carda':1, 'cardb':2}.get(oncard, 0))
+        c.write()
+        debug_print('PRS505: finished rebuild_collections')
 

src/calibre/devices/prs505/sony_cache.py

@@ -6,10 +6,8 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import os, time
-from pprint import pprint
 from base64 import b64decode
 from uuid import uuid4
-
 from lxml import etree
 
 from calibre import prints, guess_type
@@ -62,8 +60,7 @@ class XMLCache(object):
 
     def __init__(self, paths, prefixes, use_author_sort):
         if DEBUG:
-            debug_print('Building XMLCache...')
-            pprint(paths)
+            debug_print('Building XMLCache...', paths)
         self.paths = paths
         self.prefixes = prefixes
         self.use_author_sort = use_author_sort
@@ -151,15 +148,14 @@ class XMLCache(object):
                 else:
                     seen.add(title)
 
-    def get_playlist_map(self):
-        debug_print('Start get_playlist_map')
+    def build_playlist_id_map(self):
+        debug_print('Start build_playlist_id_map')
         ans = {}
         self.ensure_unique_playlist_titles()
         debug_print('after ensure_unique_playlist_titles')
         self.prune_empty_playlists()
-        debug_print('get_playlist_map loop')
         for i, root in self.record_roots.items():
-            debug_print('get_playlist_map loop', i)
+            debug_print('build_playlist_id_map loop', i)
             id_map = self.build_id_map(root)
             ans[i] = []
             for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -170,9 +166,23 @@ class XMLCache(object):
                     if record is not None:
                         items.append(record)
                 ans[i].append((playlist.get('title'), items))
-        debug_print('end get_playlist_map')
+        debug_print('end build_playlist_id_map')
         return ans
 
+    def build_id_playlist_map(self, bl_index):
+        debug_print('Start build_id_playlist_map')
+        pmap = self.build_playlist_id_map()[bl_index]
+        playlist_map = {}
+        for title, records in pmap:
+            for record in records:
+                path = record.get('path', None)
+                if path:
+                    if path not in playlist_map:
+                        playlist_map[path] = []
+                    playlist_map[path].append(title)
+        debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
+        return playlist_map
+
     def get_or_create_playlist(self, bl_idx, title):
         root = self.record_roots[bl_idx]
         for playlist in root.xpath('//*[local-name()="playlist"]'):
@@ -192,7 +202,6 @@ class XMLCache(object):
     # }}}
 
     def fix_ids(self): # {{{
-        if DEBUG:
         debug_print('Running fix_ids()')
 
         def ensure_numeric_ids(root):
@@ -276,38 +285,19 @@ class XMLCache(object):
     def update_booklist(self, bl, bl_index):
         if bl_index not in self.record_roots:
             return
-        if DEBUG:
         debug_print('Updating JSON cache:', bl_index)
+        playlist_map = self.build_id_playlist_map(bl_index)
         root = self.record_roots[bl_index]
-        pmap = self.get_playlist_map()[bl_index]
-        playlist_map = {}
-        for title, records in pmap:
-            for record in records:
-                path = record.get('path', None)
-                if path:
-                    if path not in playlist_map:
-                        playlist_map[path] = []
-                    playlist_map[path].append(title)
-
         lpath_map = self.build_lpath_map(root)
         for book in bl:
             record = lpath_map.get(book.lpath, None)
             if record is not None:
                 title = record.get('title', None)
                 if title is not None and title != book.title:
-                    if DEBUG:
                     debug_print('Renaming title', book.title, 'to', title)
                     book.title = title
-# We shouldn't do this for Sonys, because the reader strips
-# all but the first author.
-#                authors = record.get('author', None)
-#                if authors is not None:
-#                    authors = string_to_authors(authors)
-#                    if authors != book.authors:
-#                        if DEBUG:
-#                            prints('Renaming authors', book.authors, 'to',
-#                                    authors)
-#                        book.authors = authors
+                    # Don't set the author, because the reader strips all but
+                    # the first author.
                 for thumbnail in record.xpath(
                         'descendant::*[local-name()="thumbnail"]'):
                     for img in thumbnail.xpath(
@@ -318,45 +308,50 @@ class XMLCache(object):
                             book.thumbnail = raw
                             break
                     break
-                if book.lpath in playlist_map:
-                    tags = playlist_map[book.lpath]
-                    book.device_collections = tags
+                book.device_collections = playlist_map.get(book.lpath, [])
         debug_print('Finished updating JSON cache:', bl_index)
 
     # }}}
 
     # Update XML from JSON {{{
     def update(self, booklists, collections_attributes):
-        debug_print('Starting update XML from JSON')
-        playlist_map = self.get_playlist_map()
-
+        debug_print('Starting update', collections_attributes)
         for i, booklist in booklists.items():
-            if DEBUG:
+            playlist_map = self.build_id_playlist_map(i)
             debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
             lpath_map = self.build_lpath_map(root)
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
-#                record = self.book_by_lpath(book.lpath, root)
                 record = lpath_map.get(book.lpath, None)
                 if record is None:
                     record = self.create_text_record(root, i, book.lpath)
                 self.update_text_record(record, book, path, i)
+                # Ensure the collections in the XML database are recorded for
+                # this book
+                if book.device_collections is None:
+                    book.device_collections = []
+                book.device_collections = playlist_map.get(book.lpath, [])
+            self.update_playlists(i, root, booklist, collections_attributes)
+        # Update the device collections because update playlist could have added
+        # some new ones.
+        debug_print('In update/ Starting refresh of device_collections')
+        for i, booklist in booklists.items():
+            playlist_map = self.build_id_playlist_map(i)
+            for book in booklist:
+                book.device_collections = playlist_map.get(book.lpath, [])
+        self.fix_ids()
+        debug_print('Finished update')
 
-            bl_pmap = playlist_map[i]
-            self.update_playlists(i, root, booklist, bl_pmap,
-                    collections_attributes)
-
+    def rebuild_collections(self, booklist, bl_index):
+        if bl_index not in self.record_roots:
+            return
+        root = self.record_roots[bl_index]
+        self.update_playlists(bl_index, root, booklist, [])
         self.fix_ids()
 
-        # This is needed to update device_collections
-        for i, booklist in booklists.items():
-            self.update_booklist(booklist, i)
-        debug_print('Finished update XML from JSON')
-
-    def update_playlists(self, bl_index, root, booklist, playlist_map,
-            collections_attributes):
-        debug_print('Starting update_playlists')
+    def update_playlists(self, bl_index, root, booklist, collections_attributes):
+        debug_print('Starting update_playlists', collections_attributes, bl_index)
         collections = booklist.get_collections(collections_attributes)
         lpath_map = self.build_lpath_map(root)
         for category, books in collections.items():
@@ -372,10 +367,8 @@ class XMLCache(object):
                     rec.set('id', str(self.max_id(root)+1))
             ids = [x.get('id', None) for x in records]
             if None in ids:
-                if DEBUG:
                 debug_print('WARNING: Some <text> elements do not have ids')
                 ids = [x for x in ids if x is not None]
-
             playlist = self.get_or_create_playlist(bl_index, category)
             playlist_ids = []
             for item in playlist:
@@ -544,10 +537,5 @@ class XMLCache(object):
                         break
                 self.namespaces[i] = ns
 
-#        if DEBUG:
-#            debug_print('Found nsmaps:')
-#            pprint(self.nsmaps)
-#            debug_print('Found namespaces:')
-#            pprint(self.namespaces)
     # }}}
 

src/calibre/devices/usbms/books.py

@@ -11,6 +11,7 @@ from calibre.devices.mime import mime_type_ext
 from calibre.devices.interface import BookList as _BookList
 from calibre.constants import filesystem_encoding, preferred_encoding
 from calibre import isbytestring
+from calibre.utils.config import prefs
 
 class Book(MetaInformation):
 
@@ -76,7 +77,7 @@ class Book(MetaInformation):
         in C{other} takes precedence, unless the information in C{other} is NULL.
         '''
 
-        MetaInformation.smart_update(self, other)
+        MetaInformation.smart_update(self, other, replace_tags=True)
 
         for attr in self.BOOK_ATTRS:
             if hasattr(other, attr):
@@ -132,7 +133,9 @@ class CollectionsBookList(BookList):
     def get_collections(self, collection_attributes):
         collections = {}
         series_categories = set([])
-        collection_attributes = list(collection_attributes)+['device_collections']
+        collection_attributes = list(collection_attributes)
+        if prefs['preserve_user_collections']:
+            collection_attributes += ['device_collections']
         for attr in collection_attributes:
             attr = attr.strip()
             for book in self:
@@ -167,3 +170,15 @@ class CollectionsBookList(BookList):
                 books.sort(cmp=lambda x,y:cmp(getter(x), getter(y)))
         return collections
 
+    def rebuild_collections(self, booklist, oncard):
+        '''
+        For each book in the booklist for the card oncard, remove it from all
+        its current collections, then add it to the collections specified in
+        device_collections.
+
+        oncard is None for the main memory, carda for card A, cardb for card B,
+        etc.
+
+        booklist is the object created by the :method:`books` call above.
+        '''
+        pass

src/calibre/devices/usbms/device.py

@@ -78,9 +78,6 @@ class Device(DeviceConfig, DevicePlugin):
     STORAGE_CARD_VOLUME_LABEL = ''
     STORAGE_CARD2_VOLUME_LABEL = None
 
-    SUPPORTS_SUB_DIRS = False
-    MUST_READ_METADATA = False
-    SUPPORTS_USE_AUTHOR_SORT = False
 
     EBOOK_DIR_MAIN = ''
     EBOOK_DIR_CARD_A = ''

src/calibre/devices/usbms/deviceconfig.py

@@ -13,6 +13,10 @@ class DeviceConfig(object):
     EXTRA_CUSTOMIZATION_MESSAGE = None
     EXTRA_CUSTOMIZATION_DEFAULT = None
 
+    SUPPORTS_SUB_DIRS = False
+    MUST_READ_METADATA = False
+    SUPPORTS_USE_AUTHOR_SORT = False
+
     #: If None the default is used
     SAVE_TEMPLATE = None
 
@@ -23,9 +27,14 @@ class DeviceConfig(object):
             config().parse().send_template
 
     @classmethod
-    def _config(cls):
+    def _config_base_name(cls):
         klass = cls if isinstance(cls, type) else cls.__class__
-        c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
+        return klass.__name__
+
+    @classmethod
+    def _config(cls):
+        name = cls._config_base_name()
+        c = Config('device_drivers_%s' % name, _('settings for device drivers'))
         c.add_opt('format_map', default=cls.FORMATS,
                 help=_('Ordered list of formats the device will accept'))
         c.add_opt('use_subdirs', default=True,

src/calibre/ebooks/conversion/preprocess.py

@@ -107,9 +107,21 @@ class CSSPreProcessor(object):
 
     PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
 
-    def __call__(self, data):
+    def __call__(self, data, add_namespace=False):
+        from calibre.ebooks.oeb.base import XHTML_CSS_NAMESPACE
         data = self.PAGE_PAT.sub('', data)
+        if not add_namespace:
             return data
+        ans, namespaced = [], False
+        for line in data.splitlines():
+            ll = line.lstrip()
+            if not (namespaced or ll.startswith('@import') or
+                        ll.startswith('@charset')):
+                ans.append(XHTML_CSS_NAMESPACE.strip())
+                namespaced = True
+            ans.append(line)
+
+        return u'\n'.join(ans)
 
 class HTMLPreProcessor(object):
 

src/calibre/ebooks/html/input.py

@@ -20,7 +20,7 @@ from itertools import izip
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.customize.conversion import OptionRecommendation
-from calibre.constants import islinux, isfreebsd
+from calibre.constants import islinux, isfreebsd, iswindows
 from calibre import unicode_path
 from calibre.utils.localization import get_lang
 from calibre.utils.filenames import ascii_filename
@@ -32,9 +32,14 @@ class Link(object):
 
     @classmethod
     def url_to_local_path(cls, url, base):
-        path = urlunparse(('', '', url.path, url.params, url.query, ''))
+        path = url.path
+        isabs = False
+        if iswindows and path.startswith('/'):
+            path = path[1:]
+            isabs = True
+        path = urlunparse(('', '', path, url.params, url.query, ''))
         path = unquote(path)
-        if os.path.isabs(path):
+        if isabs or os.path.isabs(path):
             return path
         return os.path.abspath(os.path.join(base, path))
 
@@ -307,6 +312,7 @@ class HTMLInput(InputFormatPlugin):
             xpath
         from calibre import guess_type
         import cssutils
+        self.OEB_STYLES = OEB_STYLES
         oeb = create_oebbook(log, None, opts, self,
                 encoding=opts.input_encoding, populate=False)
         self.oeb = oeb
@@ -371,7 +377,7 @@ class HTMLInput(InputFormatPlugin):
             rewrite_links(item.data, partial(self.resource_adder, base=dpath))
 
         for item in oeb.manifest.values():
-            if item.media_type in OEB_STYLES:
+            if item.media_type in self.OEB_STYLES:
                 dpath = None
                 for path, href in self.added_resources.items():
                     if href == item.href:
@@ -409,12 +415,30 @@ class HTMLInput(InputFormatPlugin):
         oeb.container = DirContainer(os.getcwdu(), oeb.log)
         return oeb
 
+    def link_to_local_path(self, link_, base=None):
+        if not isinstance(link_, unicode):
+            try:
+                link_ = link_.decode('utf-8', 'error')
+            except:
+                self.log.warn('Failed to decode link %r. Ignoring'%link_)
+                return None, None
+        try:
+            l = Link(link_, base if base else os.getcwdu())
+        except:
+            self.log.exception('Failed to process link: %r'%link_)
+            return None, None
+        if l.path is None:
+            # Not a local resource
+            return None, None
+        link = l.path.replace('/', os.sep).strip()
+        frag = l.fragment
+        if not link:
+            return None, None
+        return link, frag
 
     def resource_adder(self, link_, base=None):
-        link = self.urlnormalize(link_)
-        link, frag = self.urldefrag(link)
-        link = unquote(link).replace('/', os.sep)
-        if not link.strip():
+        link, frag = self.link_to_local_path(link_, base=base)
+        if link is None:
             return link_
         try:
             if base and not os.path.isabs(link):
@@ -442,6 +466,9 @@ class HTMLInput(InputFormatPlugin):
 
             item = self.oeb.manifest.add(id, href, media_type)
             item.html_input_href = bhref
+            if guessed in self.OEB_STYLES:
+                item.override_css_fetch = partial(
+                        self.css_import_handler, os.path.dirname(link))
             item.data
             self.added_resources[link] = href
 
@@ -450,7 +477,17 @@ class HTMLInput(InputFormatPlugin):
             nlink = '#'.join((nlink, frag))
         return nlink
 
-
+    def css_import_handler(self, base, href):
+        link, frag = self.link_to_local_path(href, base=base)
+        if link is None or not os.access(link, os.R_OK) or os.path.isdir(link):
+            return (None, None)
+        try:
+            raw = open(link, 'rb').read().decode('utf-8', 'replace')
+            raw = self.oeb.css_preprocessor(raw, add_namespace=True)
+        except:
+            self.log.exception('Failed to read CSS file: %r'%link)
+            return (None, None)
+        return (None, raw)
 
 
 

src/calibre/ebooks/metadata/__init__.py

@@ -268,7 +268,7 @@ class MetaInformation(object):
                   ):
             prints(x, getattr(self, x, 'None'))
 
-    def smart_update(self, mi):
+    def smart_update(self, mi, replace_tags=False):
         '''
         Merge the information in C{mi} into self. In case of conflicts, the information
         in C{mi} takes precedence, unless the information in mi is NULL.
@@ -282,7 +282,7 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'language', 'guide', 'book_producer',
+                     'cover', 'guide', 'book_producer',
                      'timestamp', 'lccn', 'lcc', 'ddc', 'pubdate', 'rights',
                      'publication_type', 'uuid'):
             if hasattr(mi, attr):
@@ -291,6 +291,9 @@ class MetaInformation(object):
                     setattr(self, attr, val)
 
         if mi.tags:
+            if replace_tags:
+                self.tags = mi.tags
+            else:
                 self.tags += mi.tags
         self.tags = list(set(self.tags))
 
@@ -314,6 +317,11 @@ class MetaInformation(object):
         if len(other_comments.strip()) > len(my_comments.strip()):
             self.comments = other_comments
 
+        other_lang = getattr(mi, 'language', None)
+        if other_lang and other_lang.lower() != 'und':
+            self.language = other_lang
+
+
     def format_series_index(self):
         try:
             x = float(self.series_index)

src/calibre/ebooks/metadata/fetch.py

@@ -3,17 +3,18 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import traceback, sys, textwrap, re
+import traceback, sys, textwrap, re, urllib2
 from threading import Thread
 
-from calibre import prints
+from calibre import prints, browser
 from calibre.utils.config import OptionParser
 from calibre.utils.logging import default_log
 from calibre.customize import Plugin
+from calibre.ebooks.metadata.library_thing import OPENLIBRARY
 
 metadata_config = None
 
-class MetadataSource(Plugin):
+class MetadataSource(Plugin): # {{{
 
     author = 'Kovid Goyal'
 
@@ -130,7 +131,9 @@ class MetadataSource(Plugin):
     def customization_help(self):
         return 'This plugin can only be customized using the GUI'
 
-class GoogleBooks(MetadataSource):
+    # }}}
+
+class GoogleBooks(MetadataSource): # {{{
 
     name = 'Google Books'
     description = _('Downloads metadata from Google Books')
@@ -145,8 +148,9 @@ class GoogleBooks(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
 
-class ISBNDB(MetadataSource):
+class ISBNDB(MetadataSource): # {{{
 
     name = 'IsbnDB'
     description = _('Downloads metadata from isbndb.com')
@@ -181,7 +185,9 @@ class ISBNDB(MetadataSource):
                 'and enter your access key below.')
         return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
 
-class Amazon(MetadataSource):
+    # }}}
+
+class Amazon(MetadataSource): # {{{
 
     name = 'Amazon'
     metadata_type = 'social'
@@ -198,37 +204,27 @@ class Amazon(MetadataSource):
             self.exception = e
             self.tb = traceback.format_exc()
 
-class LibraryThing(MetadataSource):
+    # }}}
+
+class LibraryThing(MetadataSource): # {{{
 
     name = 'LibraryThing'
     metadata_type = 'social'
-    description = _('Downloads series information from librarything.com')
+    description = _('Downloads series/tags/rating information from librarything.com')
 
     def fetch(self):
         if not self.isbn:
             return
-        from calibre import browser
-        from calibre.ebooks.metadata import MetaInformation
-        import json
-        br = browser()
+        from calibre.ebooks.metadata.library_thing import get_social_metadata
         try:
-            raw = br.open(
-                    'http://status.calibre-ebook.com/library_thing/metadata/'+self.isbn
-                    ).read()
-            data = json.loads(raw)
-            if not data:
-                return
-            if 'error' in data:
-                raise Exception(data['error'])
-            if 'series' in data and 'series_index' in data:
-                mi = MetaInformation(self.title, [])
-                mi.series = data['series']
-                mi.series_index = data['series_index']
-                self.results = mi
+            self.results = get_social_metadata(self.title, self.book_author,
+                    self.publisher, self.isbn)
         except Exception, e:
             self.exception = e
             self.tb = traceback.format_exc()
 
+    # }}}
+
 
 def result_index(source, result):
     if not result.isbn:
@@ -268,6 +264,31 @@ class MetadataSources(object):
         for s in self.sources:
             s.join()
 
+def filter_metadata_results(item):
+    keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
+    for keyword in keywords:
+        if item.publisher and keyword in item.publisher.lower():
+            return False
+    return True
+
+class HeadRequest(urllib2.Request):
+    def get_method(self):
+        return "HEAD"
+
+def do_cover_check(item):
+    opener = browser()
+    item.has_cover = False
+    try:
+        opener.open(HeadRequest(OPENLIBRARY%item.isbn), timeout=5)
+        item.has_cover = True
+    except:
+        pass # Cover not found
+
+def check_for_covers(items):
+    threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
+    for t in threads: t.start()
+    for t in threads: t.join()
+
 def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
            verbose=0):
     assert not(title is None and author is None and publisher is None and \
@@ -285,10 +306,60 @@ def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
     for fetcher in fetchers[1:]:
         merge_results(results, fetcher.results)
 
-    results = sorted(results, cmp=lambda x, y : cmp(
-            (x.comments.strip() if x.comments else ''),
-            (y.comments.strip() if y.comments else '')
-                                                  ), reverse=True)
+    results = list(filter(filter_metadata_results, results))
+
+    check_for_covers(results)
+
+    words = ("the", "a", "an", "of", "and")
+    prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
+    trailing_paren_pat = re.compile(r'\(.*\)$')
+    whitespace_pat = re.compile(r'\s+')
+
+    def sort_func(x, y):
+
+        def cleanup_title(s):
+            s = s.strip().lower()
+            s = prefix_pat.sub(' ', s)
+            s = trailing_paren_pat.sub('', s)
+            s = whitespace_pat.sub(' ', s)
+            return s.strip()
+
+        t = cleanup_title(title)
+        x_title = cleanup_title(x.title)
+        y_title = cleanup_title(y.title)
+
+        # prefer titles that start with the search title
+        tx = cmp(t, x_title)
+        ty = cmp(t, y_title)
+        result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
+
+        # then prefer titles that have a cover image
+        if result == 0:
+            result = -cmp(x.has_cover, y.has_cover)
+
+        # then prefer titles with the longest comment, with in 10%
+        if result == 0:
+            cx = len(x.comments.strip() if x.comments else '')
+            cy = len(y.comments.strip() if y.comments else '')
+            t = (cx + cy) / 20
+            result = cy - cx
+            if abs(result) < t:
+                result = 0
+
+        return result
+
+    results = sorted(results, cmp=sort_func)
+
+    # if for some reason there is no comment in the top selection, go looking for one
+    if len(results) > 1:
+        if not results[0].comments or len(results[0].comments) == 0:
+            for r in results[1:]:
+                if title.lower() == r.title[:len(title)].lower() and r.comments and len(r.comments):
+                    results[0].comments = r.comments
+                    break
+
+ #   for r in results:
+ #       print "{0:14.14} {1:30.30} {2:20.20} {3:6} {4}".format(r.isbn, r.title, r.publisher, len(r.comments if r.comments else ''), r.has_cover)
 
     return results, [(x.name, x.exception, x.tb) for x in fetchers]
 

src/calibre/ebooks/metadata/library_thing.py

@@ -6,10 +6,11 @@ Fetch cover from LibraryThing.com based on ISBN number.
 
 import sys, socket, os, re
 
-from calibre import browser as _browser
+from lxml import html
+
+from calibre import browser, prints
 from calibre.utils.config import OptionParser
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
-browser = None
 
 OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
 
@@ -22,31 +23,28 @@ class ISBNNotFound(LibraryThingError):
 class ServerBusy(LibraryThingError):
     pass
 
-def login(username, password, force=True):
-    global browser
-    if browser is not None and not force:
-        return
-    browser = _browser()
-    browser.open('http://www.librarything.com')
-    browser.select_form('signup')
-    browser['formusername'] = username
-    browser['formpassword'] = password
-    browser.submit()
+def login(br, username, password, force=True):
+    br.open('http://www.librarything.com')
+    br.select_form('signup')
+    br['formusername'] = username
+    br['formpassword'] = password
+    br.submit()
 
 
 def cover_from_isbn(isbn, timeout=5., username=None, password=None):
-    global browser
-    if browser is None:
-        browser = _browser()
     src = None
+    br = browser()
     try:
-        return browser.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
+        return br.open(OPENLIBRARY%isbn, timeout=timeout).read(), 'jpg'
     except:
         pass # Cover not found
     if username and password:
-        login(username, password, force=False)
         try:
-        src = browser.open('http://www.librarything.com/isbn/'+isbn,
+            login(br, username, password, force=False)
+        except:
+            pass
+    try:
+        src = br.open_novisit('http://www.librarything.com/isbn/'+isbn,
                 timeout=timeout).read().decode('utf-8', 'replace')
     except Exception, err:
         if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
@@ -63,7 +61,7 @@ def cover_from_isbn(isbn, timeout=5., username=None, password=None):
         if url is None:
             raise LibraryThingError(_('LibraryThing.com server error. Try again later.'))
         url = re.sub(r'_S[XY]\d+', '', url['src'])
-        cover_data = browser.open(url).read()
+        cover_data = br.open_novisit(url).read()
         return cover_data, url.rpartition('.')[-1]
 
 def option_parser():
@@ -71,7 +69,7 @@ def option_parser():
 _('''
 %prog [options] ISBN
 
-Fetch a cover image for the book identified by ISBN from LibraryThing.com
+Fetch a cover image/social metadata for the book identified by ISBN from LibraryThing.com
 '''))
     parser.add_option('-u', '--username', default=None,
                       help='Username for LibraryThing.com')
@@ -79,6 +77,61 @@ Fetch a cover image for the book identified by ISBN from LibraryThing.com
                       help='Password for LibraryThing.com')
     return parser
 
+def get_social_metadata(title, authors, publisher, isbn, username=None,
+        password=None):
+    from calibre.ebooks.metadata import MetaInformation
+    mi = MetaInformation(title, authors)
+    if isbn:
+        br = browser()
+        if username and password:
+            try:
+                login(br, username, password, force=False)
+            except:
+                pass
+
+        raw = br.open_novisit('http://www.librarything.com/isbn/'
+                    +isbn).read()
+        if not raw:
+            return mi
+        root = html.fromstring(raw)
+        h1 = root.xpath('//div[@class="headsummary"]/h1')
+        if h1 and not mi.title:
+            mi.title = html.tostring(h1[0], method='text', encoding=unicode)
+        h2 = root.xpath('//div[@class="headsummary"]/h2/a')
+        if h2 and not mi.authors:
+            mi.authors = [html.tostring(x, method='text', encoding=unicode) for
+                    x in h2]
+        h3 = root.xpath('//div[@class="headsummary"]/h3/a')
+        if h3:
+            match = None
+            for h in h3:
+               series = html.tostring(h, method='text', encoding=unicode)
+               match = re.search(r'(.+) \((.+)\)', series)
+               if match is not None:
+                   break
+            if match is not None:
+                mi.series = match.group(1).strip()
+                match = re.search(r'[0-9.]+', match.group(2))
+                si = 1.0
+                if match is not None:
+                    si = float(match.group())
+                mi.series_index = si
+        tags = root.xpath('//div[@class="tags"]/span[@class="tag"]/a')
+        if tags:
+            mi.tags = [html.tostring(x, method='text', encoding=unicode) for x
+                    in tags]
+        span = root.xpath(
+                '//table[@class="wsltable"]/tr[@class="wslcontent"]/td[4]//span')
+        if span:
+            raw = html.tostring(span[0], method='text', encoding=unicode)
+            match = re.search(r'([0-9.]+)', raw)
+            if match is not None:
+                rating = float(match.group())
+                if rating > 0 and rating <= 5:
+                    mi.rating = rating
+    return mi
+
+
 def main(args=sys.argv):
     parser = option_parser()
     opts, args = parser.parse_args(args)
@@ -86,6 +139,8 @@ def main(args=sys.argv):
         parser.print_help()
         return 1
     isbn = args[1]
+    mi = get_social_metadata('', [], '', isbn)
+    prints(mi)
     cover_data, ext = cover_from_isbn(isbn, username=opts.username,
             password=opts.password)
     if not ext:

src/calibre/ebooks/metadata/opf2.py

@@ -18,7 +18,7 @@ from calibre.constants import __appname__, __version__, filesystem_encoding
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation, string_to_authors
 from calibre.utils.date import parse_date, isoformat
-
+from calibre.utils.localization import get_lang
 
 class Resource(object):
     '''
@@ -1069,7 +1069,7 @@ class OPFCreator(MetaInformation):
             dc_attrs={'id':__appname__+'_id'}))
         if getattr(self, 'pubdate', None) is not None:
             a(DC_ELEM('date', self.pubdate.isoformat()))
-        a(DC_ELEM('language', self.language if self.language else 'UND'))
+        a(DC_ELEM('language', self.language if self.language else get_lang()))
         if self.comments:
             a(DC_ELEM('description', self.comments))
         if self.publisher:
@@ -1184,7 +1184,6 @@ def metadata_to_opf(mi, as_string=True):
     factory(DC('contributor'), mi.book_producer, __appname__, 'bkp')
     if hasattr(mi.pubdate, 'isoformat'):
         factory(DC('date'), isoformat(mi.pubdate))
-    factory(DC('language'), mi.language)
     if mi.category:
         factory(DC('type'), mi.category)
     if mi.comments:
@@ -1195,6 +1194,7 @@ def metadata_to_opf(mi, as_string=True):
         factory(DC('identifier'), mi.isbn, scheme='ISBN')
     if mi.rights:
         factory(DC('rights'), mi.rights)
+    factory(DC('language'), mi.language if mi.language and mi.language.lower() != 'und' else get_lang())
     if mi.tags:
         for tag in mi.tags:
             factory(DC('subject'), tag)

src/calibre/ebooks/oeb/base.py

@@ -17,6 +17,7 @@ from urlparse import urljoin
 
 from lxml import etree, html
 from cssutils import CSSParser
+from cssutils.css import CSSRule
 
 import calibre
 from calibre.constants import filesystem_encoding
@@ -762,6 +763,7 @@ class Manifest(object):
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
+            self.override_css_fetch = None
             self.spine_position = None
             self.linear = True
             if loader is None and data is None:
@@ -982,15 +984,40 @@ class Manifest(object):
 
 
         def _parse_css(self, data):
+
+            def get_style_rules_from_import(import_rule):
+                ans = []
+                if not import_rule.styleSheet:
+                    return ans
+                rules = import_rule.styleSheet.cssRules
+                for rule in rules:
+                    if rule.type == CSSRule.IMPORT_RULE:
+                        ans.extend(get_style_rules_from_import(rule))
+                    elif rule.type in (CSSRule.FONT_FACE_RULE,
+                            CSSRule.STYLE_RULE):
+                        ans.append(rule)
+                return ans
+
             self.oeb.log.debug('Parsing', self.href, '...')
             data = self.oeb.decode(data)
-            data = self.oeb.css_preprocessor(data)
-            data = XHTML_CSS_NAMESPACE + data
+            data = self.oeb.css_preprocessor(data, add_namespace=True)
             parser = CSSParser(loglevel=logging.WARNING,
-                               fetcher=self._fetch_css,
+                               fetcher=self.override_css_fetch or self._fetch_css,
                                log=_css_logger)
             data = parser.parseString(data, href=self.href)
             data.namespaces['h'] = XHTML_NS
+            import_rules = list(data.cssRules.rulesOfType(CSSRule.IMPORT_RULE))
+            rules_to_append = []
+            insert_index = None
+            for r in data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
+                insert_index = data.cssRules.index(r)
+                break
+            for rule in import_rules:
+                rules_to_append.extend(get_style_rules_from_import(rule))
+            for r in reversed(rules_to_append):
+                data.insertRule(r, index=insert_index)
+            for rule in import_rules:
+                data.deleteRule(rule)
             return data
 
         def _fetch_css(self, path):

src/calibre/ebooks/oeb/iterator.py

@@ -139,11 +139,18 @@ class EbookIterator(object):
                     if id != -1:
                         families = [unicode(f) for f in QFontDatabase.applicationFontFamilies(id)]
                         if family:
-                            family = family.group(1).strip().replace('"', '')
-                            bad_map[family] = families[0]
-                            if family not in families:
+                            family = family.group(1)
+                            specified_families = [x.strip().replace('"',
+                                '').replace("'", '') for x in family.split(',')]
+                            aliasing_ok = False
+                            for f in specified_families:
+                                bad_map[f] = families[0]
+                                if not aliasing_ok and f in families:
+                                    aliasing_ok = True
+
+                            if not aliasing_ok:
                                 prints('WARNING: Family aliasing not fully supported.')
-                                prints('\tDeclared family: %s not in actual families: %s'
+                                prints('\tDeclared family: %r not in actual families: %r'
                                         % (family, families))
                             else:
                                 prints('Loaded embedded font:', repr(family))

src/calibre/ebooks/oeb/stylizer.py

@@ -126,6 +126,13 @@ class Stylizer(object):
             head = head[0]
         else:
             head = []
+
+        # Add cssutils parsing profiles from output_profile
+        for profile in self.opts.output_profile.extra_css_modules:
+            cssutils.profile.addProfile(profile['name'],
+                                        profile['props'],
+                                        profile['macros'])
+
         parser = cssutils.CSSParser(fetcher=self._fetch_css_file,
                 log=logging.getLogger('calibre.css'))
         self.font_face_rules = []

src/calibre/gui2/actions.py

@@ -21,6 +21,7 @@ from calibre.utils.filenames import ascii_filename
 from calibre.gui2.widgets import IMAGE_EXTENSIONS
 from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
+from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook, \
     fetch_scheduled_recipe, generate_catalog
 from calibre.constants import preferred_encoding, filesystem_encoding, \
@@ -176,6 +177,7 @@ class AnnotationsAction(object): # {{{
 
             def mark_book_as_read(self,id):
                 read_tag = gprefs.get('catalog_epub_mobi_read_tag')
+                if read_tag:
                     self.db.set_tags(id, [read_tag], append=True)
 
             def canceled(self):
@@ -830,6 +832,23 @@ class EditMetadataAction(object): # {{{
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
         # }}}
 
+    def edit_device_collections(self, view, oncard=None):
+        model = view.model()
+        result = model.get_collections_with_ids()
+        compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        d = TagListEditor(self, tag_to_match=None, data=result, compare=compare)
+        d.exec_()
+        if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old ids
+            to_delete = d.to_delete # list of ids
+            for text in to_rename:
+                for old_id in to_rename[text]:
+                    model.rename_collection(old_id, new_name=unicode(text))
+            for item in to_delete:
+                model.delete_collection_using_id(item)
+            self.upload_collections(model.db, view=view, oncard=oncard)
+            view.reset()
+
     # }}}
 
 class SaveToDiskAction(object): # {{{

src/calibre/gui2/convert/txt_input.ui

@@ -43,6 +43,9 @@
      <property name="wordWrap">
       <bool>true</bool>
      </property>
+     <property name="openExternalLinks">
+      <bool>true</bool>
+     </property>
     </widget>
    </item>
    <item row="4" column="0">

src/calibre/gui2/custom_column_widgets.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys
+import re, sys
 from functools import partial
 
 from PyQt4.Qt import QComboBox, QLabel, QSpinBox, QDoubleSpinBox, QDateEdit, \
@@ -162,7 +162,6 @@ class DateTime(Base):
             val = qt_to_dt(val)
         return val
 
-
 class Comments(Base):
 
     def setup_ui(self, parent):
@@ -199,11 +198,7 @@ class Text(Base):
             w = EnComboBox(parent)
             w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
             w.setMinimumContentsLength(25)
-
-
-
-        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
-                w]
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
 
     def initialize(self, book_id):
         val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
@@ -222,7 +217,6 @@ class Text(Base):
             if idx is not None:
                 self.widgets[1].setCurrentIndex(idx)
 
-
     def setter(self, val):
         if self.col_metadata['is_multiple']:
             if not val:
@@ -241,6 +235,58 @@ class Text(Base):
             val = None
         return val
 
+class Series(Base):
+
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel('&'+self.col_metadata['name']+_(' index:'), parent))
+        w = QDoubleSpinBox(parent)
+        w.setRange(-100., float(sys.maxint))
+        w.setDecimals(2)
+        w.setSpecialValueText(_('Undefined'))
+        w.setSingleStep(1)
+        self.idx_widget=w
+        self.widgets.append(w)
+
+    def initialize(self, book_id):
+        val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
+        s_index = self.db.get_custom_extra(book_id, num=self.col_id, index_is_id=True)
+        if s_index is None:
+            s_index = 0.0
+        self.idx_widget.setValue(s_index)
+        self.initial_index = s_index
+        self.initial_val = val
+        val = self.normalize_db_val(val)
+        idx = None
+        for i, c in enumerate(self.all_values):
+            if c == val:
+                idx = i
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+        if idx is not None:
+            self.widgets[1].setCurrentIndex(idx)
+
+    def commit(self, book_id, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        s_index = self.idx_widget.value()
+        if val != self.initial_val or s_index != self.initial_index:
+            if s_index == 0.0:
+                if tweaks['series_index_auto_increment'] == 'next':
+                    s_index = self.db.get_next_cc_series_num_for(val,
+                                                             num=self.col_id)
+                else:
+                    s_index = None
+            self.db.set_custom(book_id, val, extra=s_index,
+                               num=self.col_id, notify=notify)
+
 widgets = {
         'bool' : Bool,
         'rating' : Rating,
@@ -249,6 +295,7 @@ widgets = {
         'datetime': DateTime,
         'text' : Text,
         'comments': Comments,
+        'series': Series,
 }
 
 def field_sort(y, z, x=None):
@@ -257,35 +304,63 @@ def field_sort(y, z, x=None):
     n2 = 'zzzzz' if m2['datatype'] == 'comments' else m2['name']
     return cmp(n1.lower(), n2.lower())
 
-def populate_single_metadata_page(left, right, db, book_id, parent=None):
+def populate_metadata_page(layout, db, book_id, bulk=False, two_column=False, parent=None):
+    def widget_factory(type, col):
+        if bulk:
+            w = bulk_widgets[type](db, col, parent)
+        else:
+            w = widgets[type](db, col, parent)
+        w.initialize(book_id)
+        return w
     x = db.custom_column_num_map
     cols = list(x)
     cols.sort(cmp=partial(field_sort, x=x))
-    ans = []
-    for i, col in enumerate(cols):
-        w = widgets[x[col]['datatype']](db, col, parent)
-        ans.append(w)
-        w.initialize(book_id)
-        layout = left if i%2 == 0 else right
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
+    count_non_comment = len([c for c in cols if x[c]['datatype'] != 'comments'])
+
+    layout.setColumnStretch(1, 10)
+    if two_column:
+        turnover_point = (count_non_comment+1)/2
+        layout.setColumnStretch(3, 10)
     else:
-            w.widgets[0].setBuddy(w.widgets[1])
-            for c, widget in enumerate(w.widgets):
-                layout.addWidget(widget, row, c)
+        # Avoid problems with multi-line widgets
+        turnover_point = count_non_comment + 1000
+    ans = []
+    column = row = 0
+    for col in cols:
+        dt = x[col]['datatype']
+        if dt == 'comments':
+            continue
+        w = widget_factory(dt, col)
+        ans.append(w)
+        for c in range(0, len(w.widgets), 2):
+            w.widgets[c].setBuddy(w.widgets[c+1])
+            layout.addWidget(w.widgets[c], row, column)
+            layout.addWidget(w.widgets[c+1], row, column+1)
+            row += 1
+        if row >= turnover_point:
+            column += 2
+            turnover_point = count_non_comment + 1000
+            row = 0
+    if not bulk: # Add the comments fields
+        column = 0
+        for col in cols:
+            dt = x[col]['datatype']
+            if dt != 'comments':
+                continue
+            w = widget_factory(dt, col)
+            ans.append(w)
+            layout.addWidget(w.widgets[0], row, column, 1, 2)
+            if two_column and column == 0:
+                column = 2
+                continue
+            column = 0
+            row += 1
     items = []
     if len(ans) > 0:
         items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
             QSizePolicy.Expanding))
-        left.addItem(items[-1], left.rowCount(), 0, 1, 1)
-        left.setRowStretch(left.rowCount()-1, 100)
-    if len(ans) > 1:
-         items.append(QSpacerItem(10, 100, QSizePolicy.Minimum,
-             QSizePolicy.Expanding))
-         right.addItem(items[-1], left.rowCount(), 0, 1, 1)
-         right.setRowStretch(right.rowCount()-1, 100)
-
+        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
+        layout.setRowStretch(layout.rowCount()-1, 100)
     return ans, items
 
 class BulkBase(Base):
@@ -342,6 +417,47 @@ class BulkRating(BulkBase, Rating):
 class BulkDateTime(BulkBase, DateTime):
     pass
 
+class BulkSeries(BulkBase):
+    def setup_ui(self, parent):
+        values = self.all_values = list(self.db.all_custom(num=self.col_id))
+        values.sort(cmp = lambda x,y: cmp(x.lower(), y.lower()))
+        w = EnComboBox(parent)
+        w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon)
+        w.setMinimumContentsLength(25)
+        self.name_widget = w
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent), w]
+
+        self.widgets.append(QLabel(_('Automatically number books in this series'), parent))
+        self.idx_widget=QCheckBox(parent)
+        self.widgets.append(self.idx_widget)
+
+    def initialize(self, book_id):
+        self.idx_widget.setChecked(False)
+        for c in self.all_values:
+            self.name_widget.addItem(c)
+        self.name_widget.setEditText('')
+
+    def commit(self, book_ids, notify=False):
+        val = unicode(self.name_widget.currentText()).strip()
+        val = self.normalize_ui_val(val)
+        update_indices = self.idx_widget.checkState()
+        if val != '':
+            for book_id in book_ids:
+                if update_indices:
+                    if tweaks['series_index_auto_increment'] == 'next':
+                        s_index = self.db.get_next_cc_series_num_for\
+                                                        (val, num=self.col_id)
+                    else:
+                        s_index = 1.0
+                else:
+                    s_index = self.db.get_custom_extra(book_id, num=self.col_id,
+                                                       index_is_id=True)
+                self.db.set_custom(book_id, val, extra=s_index,
+                                   num=self.col_id, notify=notify)
+
+    def process_each_book(self):
+        return True
+
 class RemoveTags(QWidget):
 
     def __init__(self, parent, values):
@@ -431,35 +547,5 @@ bulk_widgets = {
         'float': BulkFloat,
         'datetime': BulkDateTime,
         'text' : BulkText,
+        'series': BulkSeries,
 }
-
-def populate_bulk_metadata_page(layout, db, book_ids, parent=None):
-    x = db.custom_column_num_map
-    cols = list(x)
-    cols.sort(cmp=partial(field_sort, x=x))
-    ans = []
-    for i, col in enumerate(cols):
-        dt = x[col]['datatype']
-        if dt == 'comments':
-            continue
-        w = bulk_widgets[dt](db, col, parent)
-        ans.append(w)
-        w.initialize(book_ids)
-        row = layout.rowCount()
-        if len(w.widgets) == 1:
-            layout.addWidget(w.widgets[0], row, 0, 1, -1)
-        else:
-            for c in range(0, len(w.widgets), 2):
-                w.widgets[c].setBuddy(w.widgets[c+1])
-                layout.addWidget(w.widgets[c], row, 0)
-                layout.addWidget(w.widgets[c+1], row, 1)
-                row += 1
-    items = []
-    if len(ans) > 0:
-        items.append(QSpacerItem(10, 10, QSizePolicy.Minimum,
-            QSizePolicy.Expanding))
-        layout.addItem(items[-1], layout.rowCount(), 0, 1, 1)
-        layout.setRowStretch(layout.rowCount()-1, 100)
-
-    return ans, items
-

src/calibre/gui2/device.py

@@ -10,7 +10,7 @@ from functools import partial
 from binascii import unhexlify
 
 from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \
-                     Qt, pyqtSignal, QColor, QPainter
+                     Qt, pyqtSignal, QColor, QPainter, QDialog
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre.customize.ui import available_input_formats, available_output_formats, \
@@ -294,6 +294,11 @@ class DeviceManager(Thread): # {{{
         return self.create_job(self._sync_booklists, done, args=[booklists],
                         description=_('Send metadata to device'))
 
+    def upload_collections(self, done, booklist, on_card):
+        return self.create_job(booklist.rebuild_collections, done,
+                               args=[booklist, on_card],
+                        description=_('Send collections to device'))
+
     def _upload_books(self, files, names, on_card=None, metadata=None):
         '''Upload books to device: '''
         return self.device.upload_books(files, names, on_card,
@@ -814,7 +819,8 @@ class DeviceMixin(object): # {{{
         if specific:
             d = ChooseFormatDialog(self, _('Choose format to send to device'),
                                 self.device_manager.device.settings().format_map)
-            d.exec_()
+            if d.exec_() != QDialog.Accepted:
+                return
             if d.format():
                 fmt = d.format().lower()
         dest, sub_dest = dest.split(':')
@@ -1227,6 +1233,19 @@ class DeviceMixin(object): # {{{
             return
         cp, fs = job.result
         self.location_view.model().update_devices(cp, fs)
+        # reset the views so that up-to-date info is shown. These need to be
+        # here because the sony driver updates collections in sync_booklists
+        self.memory_view.reset()
+        self.card_a_view.reset()
+        self.card_b_view.reset()
+
+    def _upload_collections(self, job):
+        if job.failed:
+            self.device_job_exception(job)
+
+    def upload_collections(self, booklist, view=None, oncard=None):
+        return self.device_manager.upload_collections(self._upload_collections,
+                                                       booklist, oncard)
 
     def upload_books(self, files, names, metadata, on_card=None, memory=None):
         '''

src/calibre/gui2/dialogs/choose_format.ui

@@ -39,7 +39,7 @@
       <enum>Qt::Horizontal</enum>
      </property>
      <property name="standardButtons" >
-      <set>QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Ok|QDialogButtonBox::Cancel</set>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/config/add_save.py

@@ -45,6 +45,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         self.metadata_box.layout().insertWidget(0, self.filename_pattern)
         self.opt_swap_author_names.setChecked(prefs['swap_author_names'])
         self.opt_add_formats_to_existing.setChecked(prefs['add_formats_to_existing'])
+        self.preserve_user_collections.setChecked(prefs['preserve_user_collections'])
         help = '\n'.join(textwrap.wrap(c.get_option('template').help, 75))
         self.save_template.initialize('save_to_disk', opts.template, help)
         self.send_template.initialize('send_to_device', opts.send_template, help)
@@ -71,6 +72,7 @@ class AddSave(QTabWidget, Ui_TabWidget):
         prefs['filename_pattern'] = pattern
         prefs['swap_author_names'] = bool(self.opt_swap_author_names.isChecked())
         prefs['add_formats_to_existing'] = bool(self.opt_add_formats_to_existing.isChecked())
+        prefs['preserve_user_collections'] = bool(self.preserve_user_collections.isChecked())
 
         return True
 

src/calibre/gui2/dialogs/config/add_save.ui

@@ -179,7 +179,31 @@ Title match ignores leading indefinite articles ("the", "a",
    </attribute>
    <layout class="QVBoxLayout" name="verticalLayout_2">
     <item>
-     <widget class="QLabel" name="label_4">
+     <widget class="QCheckBox" name="preserve_user_collections">
+      <property name="text">
+       <string>Preserve device collections.</string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_41">
+      <property name="text">
+       <string>If checked, collections will not be deleted even if a book with changed metadata is resent and the collection is not in the book's metadata. In addition, editing collections in the device view will be enabled. If unchecked, collections will be always reflect only the metadata in the calibre library.</string>
+      </property>
+      <property name="wordWrap">
+       <bool>true</bool>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_42">
+      <property name="text">
+       <string>  </string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QLabel" name="label_43">
       <property name="text">
        <string>Here you can control how calibre will save your books when you click the Send to Device button. This setting can be overriden for individual devices by customizing the device interface plugins in Preferences-&gt;Plugins</string>
       </property>

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -24,16 +24,19 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
                     2:{'datatype':'comments',
                         'text':_('Long text, like comments, not shown in the tag browser'),
                         'is_multiple':False},
-                    3:{'datatype':'datetime',
+                    3:{'datatype':'series',
+                        'text':_('Text column for keeping series-like information'),
+                        'is_multiple':False},
+                    4:{'datatype':'datetime',
                         'text':_('Date'), 'is_multiple':False},
-                    4:{'datatype':'float',
+                    5:{'datatype':'float',
                         'text':_('Floating point numbers'), 'is_multiple':False},
-                    5:{'datatype':'int',
+                    6:{'datatype':'int',
                         'text':_('Integers'), 'is_multiple':False},
-                    6:{'datatype':'rating',
+                    7:{'datatype':'rating',
                         'text':_('Ratings, shown with stars'),
                         'is_multiple':False},
-                    7:{'datatype':'bool',
+                    8:{'datatype':'bool',
                         'text':_('Yes/No'), 'is_multiple':False},
                 }
 

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -10,7 +10,7 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.ebooks.metadata import string_to_authors, \
     authors_to_string
-from calibre.gui2.custom_column_widgets import populate_bulk_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
 
@@ -44,15 +44,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             self.central_widget.tabBar().setVisible(False)
         else:
             self.create_custom_column_editors()
-
         self.exec_()
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
         layout = QGridLayout()
-
-        self.custom_column_widgets, self.__cc_spacers = populate_bulk_metadata_page(
-                layout, self.db, self.ids, w)
+        self.custom_column_widgets, self.__cc_spacers = \
+            populate_metadata_page(layout, self.db, self.ids, parent=w,
+                                   two_column=False, bulk=True)
         w.setLayout(layout)
         self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets

src/calibre/gui2/dialogs/metadata_single.py

@@ -32,7 +32,7 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.date import qt_to_dt
 from calibre.customize.ui import run_plugins_on_import, get_isbndb_key
 from calibre.gui2.dialogs.config.social import SocialMetadata
-from calibre.gui2.custom_column_widgets import populate_single_metadata_page
+from calibre.gui2.custom_column_widgets import populate_metadata_page
 
 class CoverFetcher(QThread):
 
@@ -420,23 +420,19 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
 
     def create_custom_column_editors(self):
         w = self.central_widget.widget(1)
-        top_layout = QHBoxLayout()
-        top_layout.setSpacing(20)
-        left_layout = QGridLayout()
-        right_layout = QGridLayout()
-        top_layout.addLayout(left_layout)
-
-        self.custom_column_widgets, self.__cc_spacers = populate_single_metadata_page(
-                left_layout, right_layout, self.db, self.id, w)
-        top_layout.addLayout(right_layout)
-        sip.delete(w.layout())
-        w.setLayout(top_layout)
-        self.__custom_col_layouts = [top_layout, left_layout, right_layout]
+        layout = w.layout()
+        self.custom_column_widgets, self.__cc_spacers = \
+                    populate_metadata_page(layout, self.db, self.id,
+                                           parent=w, bulk=False, two_column=True)
+        self.__custom_col_layouts = [layout]
         ans = self.custom_column_widgets
         for i in range(len(ans)-1):
-            w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[-1])
-
-
+            if len(ans[i+1].widgets) == 2:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1])
+            else:
+                w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0])
+            for c in range(2, len(ans[i].widgets), 2):
+                w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1])
 
     def validate_isbn(self, isbn):
         isbn = unicode(isbn).strip()

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,51 +1,61 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from functools import partial
 from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog
-from calibre.ebooks.metadata import title_sort
+
+class ListWidgetItem(QListWidgetItem):
+
+    def __init__(self, txt):
+        QListWidgetItem.__init__(self, txt)
+        self.old_value = txt
+        self.cur_value = txt
+
+    def data(self, role):
+        if role == Qt.DisplayRole:
+            if self.old_value != self.cur_value:
+                return _('%s (was %s)'%(self.cur_value, self.old_value))
+            else:
+                return self.cur_value
+        elif role == Qt.EditRole:
+            return self.cur_value
+        else:
+            return QListWidgetItem.data(self, role)
+
+    def setData(self, role, data):
+        if role == Qt.EditRole:
+            self.cur_value = data.toString()
+        QListWidgetItem.setData(self, role, data)
+
+    def text(self):
+        return self.cur_value
+
+    def setText(self, txt):
+        self.cur_value = txt
+        QListWidgetItem.setText(txt)
 
 class TagListEditor(QDialog, Ui_TagListEditor):
 
-    def __init__(self, window, db, tag_to_match, category):
+    def __init__(self, window, tag_to_match, data, compare):
         QDialog.__init__(self, window)
         Ui_TagListEditor.__init__(self)
         self.setupUi(self)
 
         self.to_rename = {}
         self.to_delete = []
-        self.db = db
         self.all_tags = {}
-        self.category = category
-        if category == 'tags':
-            result = db.get_tags_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        elif category == 'series':
-            result = db.get_series_with_ids()
-            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
-        elif category == 'publisher':
-            result = db.get_publishers_with_ids()
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
-        else: # should be a custom field
-            self.cc_label = None
-            if category in db.field_metadata:
-                self.cc_label = db.field_metadata[category]['label']
-                result = self.db.get_custom_items_with_ids(label=self.cc_label)
-            else:
-                result = []
-            compare = (lambda x,y:cmp(x.lower(), y.lower()))
 
-        for k,v in result:
+        for k,v in data:
             self.all_tags[v] = k
         for tag in sorted(self.all_tags.keys(), cmp=compare):
-            item = QListWidgetItem(tag)
+            item = ListWidgetItem(tag)
             item.setData(Qt.UserRole, self.all_tags[tag])
             self.available_tags.addItem(item)
 
+        if tag_to_match is not None:
             items = self.available_tags.findItems(tag_to_match, Qt.MatchExactly)
             if len(items) == 1:
                 self.available_tags.setCurrentItem(items[0])
@@ -62,13 +72,11 @@ class TagListEditor(QDialog, Ui_TagListEditor):
                 item.setText(self.item_before_editing.text())
                 return
         if item.text() != self.item_before_editing.text():
-            if item.text() in self.all_tags.keys() or item.text() in self.to_rename.keys():
-                error_dialog(self, _('Item already used'),
-                             _('The item %s is already used.')%(item.text())).exec_()
-                item.setText(self.item_before_editing.text())
-                return
             (id,ign) = self.item_before_editing.data(Qt.UserRole).toInt()
-            self.to_rename[item.text()] = id
+            if item.text() not in self.to_rename:
+                self.to_rename[item.text()] = [id]
+            else:
+                self.to_rename[item.text()].append(id)
 
     def rename_tag(self):
         item = self.available_tags.currentItem()
@@ -99,30 +107,3 @@ class TagListEditor(QDialog, Ui_TagListEditor):
             self.to_delete.append(id)
             self.available_tags.takeItem(self.available_tags.row(item))
 
-    def accept(self):
-        rename_func = None
-        if self.category == 'tags':
-            rename_func = self.db.rename_tag
-            delete_func = self.db.delete_tag_using_id
-        elif self.category == 'series':
-            rename_func = self.db.rename_series
-            delete_func = self.db.delete_series_using_id
-        elif self.category == 'publisher':
-            rename_func = self.db.rename_publisher
-            delete_func = self.db.delete_publisher_using_id
-        else:
-            rename_func = partial(self.db.rename_custom_item, label=self.cc_label)
-            delete_func = partial(self.db.delete_custom_item_using_id, label=self.cc_label)
-
-        work_done = False
-        if rename_func:
-            for text in self.to_rename:
-                work_done = True
-                rename_func(id=self.to_rename[text], new_name=unicode(text))
-            for item in self.to_delete:
-                work_done = True
-                delete_func(item)
-        if not work_done:
-            QDialog.reject(self)
-        else:
-            QDialog.accept(self)

src/calibre/gui2/init.py

@@ -226,17 +226,30 @@ class LibraryViewMixin(object): # {{{
                                         self.action_show_book_details,
                                         self.action_del,
                                         add_to_library = None,
+                                        edit_device_collections=None,
                                         similar_menu=similar_menu)
         add_to_library = (_('Add books to library'), self.add_books_from_device)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard=None))
         self.memory_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='carda'))
         self.card_a_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
+
+        edit_device_collections = (_('Manage collections'),
+                            partial(self.edit_device_collections, oncard='cardb'))
         self.card_b_view.set_context_menu(None, None, None,
                 self.action_view, self.action_save, None, None, self.action_del,
-                add_to_library=add_to_library)
+                add_to_library=add_to_library,
+                edit_device_collections=edit_device_collections)
 
         self.library_view.files_dropped.connect(self.files_dropped, type=Qt.QueuedConnection)
         for func, args in [
@@ -249,9 +262,14 @@ class LibraryViewMixin(object): # {{{
                 getattr(view, func)(*args)
 
         self.memory_view.connect_dirtied_signal(self.upload_booklists)
+        self.memory_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard=None)
         self.card_a_view.connect_dirtied_signal(self.upload_booklists)
+        self.card_a_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='carda')
         self.card_b_view.connect_dirtied_signal(self.upload_booklists)
-
+        self.card_b_view.connect_upload_collections_signal(
+                                    func=self.upload_collections, oncard='cardb')
         self.book_on_device(None, reset=True)
         db.set_book_on_device_func(self.book_on_device)
         self.library_view.set_database(db)

src/calibre/gui2/library/models.py

@@ -16,11 +16,12 @@ from calibre.gui2 import NONE, config, UNDEFINED_QDATE
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_authors
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
 from calibre.gui2.library import DEFAULT_SORT
@@ -520,7 +521,7 @@ class BooksModel(QAbstractTableModel): # {{{
                 return QVariant(', '.join(sorted(tags.split(','))))
             return None
 
-        def series(r, idx=-1, siix=-1):
+        def series_type(r, idx=-1, siix=-1):
             series = self.db.data[r][idx]
             if series:
                 idx = fmt_sidx(self.db.data[r][siix])
@@ -591,7 +592,7 @@ class BooksModel(QAbstractTableModel): # {{{
                                 idx=self.db.field_metadata['publisher']['rec_index'], mult=False),
                    'tags'     : functools.partial(tags,
                                 idx=self.db.field_metadata['tags']['rec_index']),
-                   'series'   : functools.partial(series,
+                   'series'   : functools.partial(series_type,
                                 idx=self.db.field_metadata['series']['rec_index'],
                                 siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
@@ -620,6 +621,9 @@ class BooksModel(QAbstractTableModel): # {{{
                                             bool_cols_are_tristate=tweaks['bool_custom_columns_are_tristate'] == 'yes')
             elif datatype == 'rating':
                 self.dc[col] = functools.partial(rating_type, idx=idx)
+            elif datatype == 'series':
+                self.dc[col] = functools.partial(series_type, idx=idx,
+                    siix=self.db.field_metadata.cc_series_index_column_for(col))
             else:
                 print 'What type is this?', col, datatype
         # build a index column to data converter map, to remove the string lookup in the data loop
@@ -681,6 +685,8 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_custom_column_data(self, row, colhead, value):
         typ = self.custom_columns[colhead]['datatype']
+        label=self.db.field_metadata.key_to_label(colhead)
+        s_index = None
         if typ in ('text', 'comments'):
             val = unicode(value.toString()).strip()
             val = val if val else None
@@ -702,9 +708,10 @@ class BooksModel(QAbstractTableModel): # {{{
                 if not val.isValid():
                     return False
                 val = qt_to_dt(val, as_utc=False)
-        self.db.set_custom(self.db.id(row), val,
-                           label=self.db.field_metadata.key_to_label(colhead),
-                           num=None, append=False, notify=True)
+        elif typ == 'series':
+            val, s_index = parse_series_string(self.db, label, value.toString())
+        self.db.set_custom(self.db.id(row), val, extra=s_index,
+                           label=label, num=None, append=False, notify=True)
         return True
 
     def setData(self, index, value, role):
@@ -850,6 +857,7 @@ class OnDeviceSearch(SearchQueryParser): # {{{
 class DeviceBooksModel(BooksModel): # {{{
 
     booklist_dirtied = pyqtSignal()
+    upload_collections = pyqtSignal(object)
 
     def __init__(self, parent):
         BooksModel.__init__(self, parent)
@@ -920,11 +928,12 @@ class DeviceBooksModel(BooksModel): # {{{
         if index.isValid() and self.editable:
             cname = self.column_map[index.column()]
             if cname in ('title', 'authors') or \
-                    (cname == 'collections' and self.db.supports_collections()):
+                    (cname == 'collections' and \
+                     self.db.supports_collections() and \
+                     prefs['preserve_user_collections']):
                 flags |= Qt.ItemIsEditable
         return flags
 
-
     def search(self, text, reset=True):
         if not text or not text.strip():
             self.map = list(range(len(self.db)))
@@ -970,8 +979,8 @@ class DeviceBooksModel(BooksModel): # {{{
             x, y = int(self.db[x].size), int(self.db[y].size)
             return cmp(x, y)
         def tagscmp(x, y):
-            x = ','.join(self.db[x].device_collections)
-            y = ','.join(self.db[y].device_collections)
+            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []))).lower()
+            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []))).lower()
             return cmp(x, y)
         def libcmp(x, y):
             x, y = self.db[x].in_library, self.db[y].in_library
@@ -1072,6 +1081,36 @@ class DeviceBooksModel(BooksModel): # {{{
                 res.append((r,b))
         return res
 
+    def get_collections_with_ids(self):
+        collections = set()
+        for book in self.db:
+            if book.device_collections is not None:
+                collections.update(set(book.device_collections))
+        self.collections = []
+        result = []
+        for i,collection in enumerate(collections):
+            result.append((i, collection))
+            self.collections.append(collection)
+        return result
+
+    def rename_collection(self, old_id, new_name):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+                if new_name not in book.device_collections:
+                    book.device_collections.append(new_name)
+
+    def delete_collection_using_id(self, old_id):
+        old_name = self.collections[old_id]
+        for book in self.db:
+            if book.device_collections is None:
+                continue
+            if old_name in book.device_collections:
+                book.device_collections.remove(old_name)
+
     def indices(self, rows):
         '''
         Return indices into underlying database from rows
@@ -1102,6 +1141,7 @@ class DeviceBooksModel(BooksModel): # {{{
             elif cname == 'collections':
                 tags = self.db[self.map[row]].device_collections
                 if tags:
+                    tags.sort(cmp=lambda x,y: cmp(x.lower(), y.lower()))
                     return QVariant(', '.join(tags))
         elif role == Qt.ToolTipRole and index.isValid():
             if self.map[row] in self.indices_to_be_deleted():
@@ -1144,14 +1184,18 @@ class DeviceBooksModel(BooksModel): # {{{
                 return False
             val = unicode(value.toString()).strip()
             idx = self.map[row]
+            if cname == 'collections':
+                tags = [i.strip() for i in val.split(',')]
+                tags = [t for t in tags if t]
+                self.db[idx].device_collections = tags
+                self.dataChanged.emit(index, index)
+                self.upload_collections.emit(self.db)
+                return True
+
             if cname == 'title' :
                 self.db[idx].title = val
             elif cname == 'authors':
                 self.db[idx].authors = string_to_authors(val)
-            elif cname == 'collections':
-                tags = [i.strip() for i in val.split(',')]
-                tags = [t for t in tags if t]
-                self.db[idx].device_collections = tags
             self.dataChanged.emit(index, index)
             self.booklist_dirtied.emit()
             done = True

src/calibre/gui2/library/views.py

@@ -15,7 +15,7 @@ from calibre.gui2.library.delegates import RatingDelegate, PubDateDelegate, \
     TextDelegate, DateDelegate, TagsDelegate, CcTextDelegate, \
     CcBoolDelegate, CcCommentsDelegate, CcDateDelegate
 from calibre.gui2.library.models import BooksModel, DeviceBooksModel
-from calibre.utils.config import tweaks
+from calibre.utils.config import tweaks, prefs
 from calibre.gui2 import error_dialog, gprefs
 from calibre.gui2.library import DEFAULT_SORT
 
@@ -347,7 +347,7 @@ class BooksView(QTableView): # {{{
                     self.setItemDelegateForColumn(cm.index(colhead), delegate)
                 elif cc['datatype'] == 'comments':
                     self.setItemDelegateForColumn(cm.index(colhead), self.cc_comments_delegate)
-                elif cc['datatype'] == 'text':
+                elif cc['datatype'] in ('text', 'series'):
                     if cc['is_multiple']:
                         self.setItemDelegateForColumn(cm.index(colhead), self.tags_delegate)
                     else:
@@ -371,7 +371,8 @@ class BooksView(QTableView): # {{{
     # Context Menu {{{
     def set_context_menu(self, edit_metadata, send_to_device, convert, view,
                          save, open_folder, book_details, delete,
-                         similar_menu=None, add_to_library=None):
+                         similar_menu=None, add_to_library=None,
+                         edit_device_collections=None):
         self.setContextMenuPolicy(Qt.DefaultContextMenu)
         self.context_menu = QMenu(self)
         if edit_metadata is not None:
@@ -393,6 +394,10 @@ class BooksView(QTableView): # {{{
         if add_to_library is not None:
             func = partial(add_to_library[1], view=self)
             self.context_menu.addAction(add_to_library[0], func)
+        if edit_device_collections is not None:
+            func = partial(edit_device_collections[1], view=self)
+            self.edit_collections_menu = \
+                self.context_menu.addAction(edit_device_collections[0], func)
 
     def contextMenuEvent(self, event):
         self.context_menu.popup(event.globalPos())
@@ -494,6 +499,13 @@ class DeviceBooksView(BooksView): # {{{
         self.setDragDropMode(self.NoDragDrop)
         self.setAcceptDrops(False)
 
+    def contextMenuEvent(self, event):
+        self.edit_collections_menu.setVisible(
+                            self._model.db.supports_collections() and \
+                            prefs['preserve_user_collections'])
+        self.context_menu.popup(event.globalPos())
+        event.accept()
+
     def set_database(self, db):
         self._model.set_database(db)
         self.restore_state()
@@ -505,6 +517,9 @@ class DeviceBooksView(BooksView): # {{{
     def connect_dirtied_signal(self, slot):
         self._model.booklist_dirtied.connect(slot)
 
+    def connect_upload_collections_signal(self, func=None, oncard=None):
+        self._model.upload_collections.connect(partial(func, view=self, oncard=oncard))
+
     def dropEvent(self, *args):
         error_dialog(self, _('Not allowed'),
         _('Dropping onto a device is not supported. First add the book to the calibre library.')).exec_()

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -75,10 +75,6 @@
 
 #include <QDebug>
 
-// uncomment this to enable bilinear filtering for texture mapping
-// gives much better rendering, at the cost of memory space
-// #define PICTUREFLOW_BILINEAR_FILTER
-
 // for fixed-point arithmetic, we need minimum 32-bit long
 // long long (64-bit) might be useful for multiplication and division
 typedef long PFreal;
@@ -376,7 +372,6 @@ private:
   int slideWidth;
   int slideHeight;
   int fontSize;
-  int zoom;
   int queueLength;
 
   int centerIndex;
@@ -401,6 +396,7 @@ private:
 
   void recalc(int w, int h);
   QRect renderSlide(const SlideInfo &slide, int alpha=256, int col1=-1, int col=-1);
+  QRect renderCenterSlide(const SlideInfo &slide);
   QImage* surface(int slideIndex);
   void triggerRender();
   void resetSlides();
@@ -414,7 +410,6 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_)
   slideWidth = 200;
   slideHeight = 200;
   fontSize = 10;
-  zoom = 100;
 
   centerIndex = 0;
   queueLength = queueLength_;
@@ -464,21 +459,6 @@ void PictureFlowPrivate::setSlideSize(QSize size)
   triggerRender();
 }
 
-int PictureFlowPrivate::zoomFactor() const
-{
-  return zoom;
-}
-
-void PictureFlowPrivate::setZoomFactor(int z)
-{
-  if(z <= 0)
-    return;
-
-  zoom = z;
-  recalc(buffer.width(), buffer.height());
-  triggerRender();
-}
-
 QImage PictureFlowPrivate::slide(int index) const
 {
   return slideImages->image(index);
@@ -554,7 +534,8 @@ void PictureFlowPrivate::resize(int w, int h)
   if (w < 10) w = 10;
   if (h < 10) h = 10;
   slideHeight = int(float(h)/REFLECTION_FACTOR);
-  slideWidth = int(float(slideHeight) * 2/3.);
+  slideWidth = int(float(slideHeight) * 3./4.);
+  //qDebug() << slideHeight << "x" << slideWidth;
   fontSize = MAX(int(h/15.), 12);
   recalc(w, h);
   resetSlides();
@@ -595,15 +576,12 @@ void PictureFlowPrivate::resetSlides()
   }
 }
 
-#define BILINEAR_STRETCH_HOR 4
-#define BILINEAR_STRETCH_VER 4
-
 static QImage prepareSurface(QImage img, int w, int h)
 {
   Qt::TransformationMode mode = Qt::SmoothTransformation;
-  img = img.scaled(w, h, Qt::IgnoreAspectRatio, mode);
+  img = img.scaled(w, h, Qt::KeepAspectRatioByExpanding, mode);
 
-  // slightly larger, to accomodate for the reflection
+  // slightly larger, to accommodate for the reflection
   int hs = int(h * REFLECTION_FACTOR);
   int hofs = 0;
 
@@ -633,12 +611,6 @@ static QImage prepareSurface(QImage img, int w, int h)
       result.setPixel(h+hofs+y, x, qRgb(r, g, b));
     }
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int hh = BILINEAR_STRETCH_VER*hs;
-  int ww = BILINEAR_STRETCH_HOR*w;
-  result = result.scaled(hh, ww, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
-#endif
-
   return result;
 }
 
@@ -699,8 +671,12 @@ void PictureFlowPrivate::render()
 
   int nleft = leftSlides.count();
   int nright = rightSlides.count();
+  QRect r;
 
-  QRect r = renderSlide(centerSlide);
+  if (step == 0) 
+      r = renderCenterSlide(centerSlide);
+  else
+      r = renderSlide(centerSlide);
   int c1 = r.left();
   int c2 = r.right();
 
@@ -813,7 +789,23 @@ static inline uint BYTE_MUL_RGB16_32(uint x, uint a) {
     return t;
 }
 
+QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) {
+  QImage* src = surface(slide.slideIndex);
+  if(!src)
+    return QRect();
 
+  int sw = src->height();
+  int sh = src->width();
+  int h = buffer.height();
+  QRect rect(buffer.width()/2 - sw/2, 0, sw, h-1);
+  int left = rect.left();
+
+  for(int x = 0; x < sh-1; x++)
+    for(int y = 0; y < sw; y++)
+      buffer.setPixel(left + y, 1+x, src->pixel(x, y));
+
+  return rect;
+}
 // Renders a slide to offscreen buffer. Returns a rect of the rendered area.
 // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent
 // col1 and col2 limit the column for rendering.
@@ -826,13 +818,8 @@ int col1, int col2)
 
   QRect rect(0, 0, 0, 0);  
   
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-  int sw = src->height() / BILINEAR_STRETCH_HOR;
-  int sh = src->width() / BILINEAR_STRETCH_VER;
-#else
   int sw = src->height();
   int sh = src->width();
-#endif
   int h = buffer.height();
   int w = buffer.width();
 
@@ -848,7 +835,7 @@ int col1, int col2)
   col1 = qMin(col1, w-1);
   col2 = qMin(col2, w-1);
 
-  int distance = h * 100 / zoom;
+  int distance = h;
   PFreal sdx = fcos(slide.angle);
   PFreal sdy = fsin(slide.angle);
   PFreal xs = slide.cx - slideWidth * sdx/2;
@@ -878,15 +865,9 @@ int col1, int col2)
     PFreal hitx = fmul(dist, rays[x]);
     PFreal hitdist = fdiv(hitx - slide.cx, sdx);
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int column = sw*BILINEAR_STRETCH_HOR/2 + (hitdist*BILINEAR_STRETCH_HOR >> PFREAL_SHIFT);
-    if(column >= sw*BILINEAR_STRETCH_HOR)
-      break;
-#else
     int column = sw/2 + (hitdist >> PFREAL_SHIFT);
     if(column >= sw)
       break;
-#endif
     if(column < 0)
       continue;
 
@@ -901,13 +882,8 @@ int col1, int col2)
     QRgb565* pixel2 = (QRgb565*)(buffer.scanLine(y2)) + x;
     int pixelstep = pixel2 - pixel1;
 
-#ifdef PICTUREFLOW_BILINEAR_FILTER
-    int center = (sh*BILINEAR_STRETCH_VER/2);
-    int dy = dist*BILINEAR_STRETCH_VER / h;
-#else
     int center = sh/2;
     int dy = dist / h;
-#endif
     int p1 = center*PFREAL_ONE - dy/2;
     int p2 = center*PFREAL_ONE + dy/2;
 
@@ -1155,16 +1131,6 @@ void PictureFlow::setSlideSize(QSize size)
   d->setSlideSize(size);
 }
 
-int PictureFlow::zoomFactor() const
-{
-  return d->zoomFactor();
-}
-
-void PictureFlow::setZoomFactor(int z)
-{
-  d->setZoomFactor(z);
-}
-
 QImage PictureFlow::slide(int index) const
 {
   return d->slide(index);

src/calibre/gui2/pictureflow/pictureflow.h

@@ -91,7 +91,6 @@ Q_OBJECT
 
   Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide)
   Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize)
-  Q_PROPERTY(int zoomFactor READ zoomFactor WRITE setZoomFactor)
 
 public:
   /*!
@@ -120,16 +119,6 @@ public:
   */  
   void setSlideSize(QSize size);
 
-  /*!
-    Sets the zoom factor (in percent).
-  */ 
-  void setZoomFactor(int zoom);
-
-  /*!
-    Returns the zoom factor (in percent).
-  */
-  int zoomFactor() const;
-
   /*!
     Clears any caches held to free up memory
   */

src/calibre/gui2/pictureflow/pictureflow.sip

@@ -40,10 +40,6 @@ public :
 
   void setSlideSize(QSize size);
 
-  void setZoomFactor(int zoom);
-
-  int zoomFactor() const;
-
   void clearCaches();
 
   virtual QImage slide(int index) const;

src/calibre/gui2/search_box.py

@@ -56,7 +56,8 @@ class SearchBox2(QComboBox):
     To use this class:
 
         * Call initialize()
-        * Connect to the search() and cleared() signals from this widget
+        * Connect to the search() and cleared() signals from this widget.
+        * Connect to the cleared() signal to know when the box content changes
         * Call search_done() after every search is complete
         * Use clear() to clear back to the help message
     '''
@@ -75,6 +76,7 @@ class SearchBox2(QComboBox):
                 type=Qt.DirectConnection)
         self.line_edit.mouse_released.connect(self.mouse_released,
                 type=Qt.DirectConnection)
+        self.activated.connect(self.history_selected)
         self.setEditable(True)
         self.help_state = False
         self.as_you_type = True
@@ -139,6 +141,9 @@ class SearchBox2(QComboBox):
 
     def key_pressed(self, event):
         self.normalize_state()
+        if self._in_a_search:
+            self.emit(SIGNAL('changed()'))
+            self._in_a_search = False
         if event.key() in (Qt.Key_Return, Qt.Key_Enter):
             self.do_search()
         self.timer = self.startTimer(self.__class__.INTERVAL)
@@ -154,6 +159,10 @@ class SearchBox2(QComboBox):
             self.timer = None
             self.do_search()
 
+    def history_selected(self, text):
+        self.emit(SIGNAL('changed()'))
+        self.do_search()
+
     @property
     def smart_text(self):
         text = unicode(self.currentText()).strip()
@@ -345,6 +354,7 @@ class SearchBoxMixin(object):
         self.search.initialize('main_search_history', colorize=True,
                 help_text=_('Search (For Advanced Search click the button to the left)'))
         self.connect(self.search, SIGNAL('cleared()'), self.search_box_cleared)
+        self.connect(self.search, SIGNAL('changed()'), self.search_box_changed)
         self.connect(self.clear_button, SIGNAL('clicked()'), self.search.clear)
         QObject.connect(self.advanced_search_button, SIGNAL('clicked(bool)'),
                         self.do_advanced_search)
@@ -364,6 +374,9 @@ class SearchBoxMixin(object):
         self.saved_search.clear_to_help()
         self.set_number_of_books_shown()
 
+    def search_box_changed(self):
+        self.tags_view.clear()
+
     def do_advanced_search(self, *args):
         d = SearchDialog(self)
         if d.exec_() == QDialog.Accepted:

src/calibre/gui2/tag_view.py

@@ -15,6 +15,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
                      QAbstractItemModel, QVariant, QModelIndex, QMenu, \
                      QPushButton, QWidget, QItemDelegate
 
+from calibre.ebooks.metadata import title_sort
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
 from calibre.library.field_metadata import TagsIcons
@@ -680,9 +681,50 @@ class TagBrowserMixin(object): # {{{
             self.tags_view.recount()
 
     def do_tags_list_edit(self, tag, category):
-        d = TagListEditor(self, self.library_view.model().db, tag, category)
+        db=self.library_view.model().db
+        if category == 'tags':
+            result = db.get_tags_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        elif category == 'series':
+            result = db.get_series_with_ids()
+            compare = (lambda x,y:cmp(title_sort(x).lower(), title_sort(y).lower()))
+        elif category == 'publisher':
+            result = db.get_publishers_with_ids()
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+        else: # should be a custom field
+            cc_label = None
+            if category in db.field_metadata:
+                cc_label = db.field_metadata[category]['label']
+                result = self.db.get_custom_items_with_ids(label=cc_label)
+            else:
+                result = []
+            compare = (lambda x,y:cmp(x.lower(), y.lower()))
+
+        d = TagListEditor(self, tag_to_match=tag, data=result, compare=compare)
         d.exec_()
         if d.result() == d.Accepted:
+            to_rename = d.to_rename # dict of new text to old id
+            to_delete = d.to_delete # list of ids
+            rename_func = None
+            if category == 'tags':
+                rename_func = db.rename_tag
+                delete_func = db.delete_tag_using_id
+            elif category == 'series':
+                rename_func = db.rename_series
+                delete_func = db.delete_series_using_id
+            elif category == 'publisher':
+                rename_func = db.rename_publisher
+                delete_func = db.delete_publisher_using_id
+            else:
+                rename_func = partial(db.rename_custom_item, label=cc_label)
+                delete_func = partial(db.delete_custom_item_using_id, label=cc_label)
+            if rename_func:
+                for text in to_rename:
+                        for old_id in to_rename[text]:
+                            rename_func(old_id, new_name=unicode(text))
+                for item in to_delete:
+                    delete_func(item)
+
             # Clean up everything, as information could have changed for many books.
             self.library_view.model().refresh()
             self.tags_view.set_new_model()

src/calibre/gui2/ui.py

@@ -473,6 +473,8 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
             self.search_restriction.setEnabled(False)
             for action in list(self.delete_menu.actions())[1:]:
                 action.setEnabled(False)
+            # Reset the view in case something changed while it was invisible
+            self.current_view().reset()
         self.set_number_of_books_shown()
 
 

src/calibre/gui2/widgets.py

@@ -957,16 +957,19 @@ class LayoutButton(QToolButton):
 
         self.splitter = splitter
         splitter.state_changed.connect(self.update_state)
+        self.setCursor(Qt.PointingHandCursor)
 
     def set_state_to_show(self, *args):
         self.setChecked(False)
         label =_('Show')
         self.setText(label + ' ' + self.label)
+        self.setToolTip(self.text())
 
     def set_state_to_hide(self, *args):
         self.setChecked(True)
         label = _('Hide')
         self.setText(label + ' ' + self.label)
+        self.setToolTip(self.text())
 
     def update_state(self, *args):
         if self.splitter.is_side_index_hidden:

src/calibre/library/caches.py

@@ -401,7 +401,8 @@ class ResultCache(SearchQueryParser):
             for x in self.field_metadata:
                 if len(self.field_metadata[x]['search_terms']):
                     db_col[x] = self.field_metadata[x]['rec_index']
-                    if self.field_metadata[x]['datatype'] not in ['text', 'comments']:
+                    if self.field_metadata[x]['datatype'] not in \
+                                                ['text', 'comments', 'series']:
                         exclude_fields.append(db_col[x])
                     col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
                     is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
@@ -580,16 +581,18 @@ class ResultCache(SearchQueryParser):
             self.sort(field, ascending)
         self._map_filtered = list(self._map)
 
-    def seriescmp(self, x, y):
-        sidx = self.FIELD_MAP['series']
+    def seriescmp(self, sidx, siidx, x, y, library_order=None):
         try:
+            if library_order:
                 ans = cmp(title_sort(self._data[x][sidx].lower()),
                                 title_sort(self._data[y][sidx].lower()))
+            else:
+                ans = cmp(self._data[x][sidx].lower(),
+                                                self._data[y][sidx].lower())
         except AttributeError: # Some entries may be None
             ans = cmp(self._data[x][sidx], self._data[y][sidx])
         if ans != 0: return ans
-        sidx = self.FIELD_MAP['series_index']
-        return cmp(self._data[x][sidx], self._data[y][sidx])
+        return cmp(self._data[x][siidx], self._data[y][siidx])
 
     def cmp(self, loc, x, y, asstr=True, subsort=False):
         try:
@@ -617,18 +620,27 @@ class ResultCache(SearchQueryParser):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
-        if self.field_metadata[field]['is_custom']:
-            as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-            field = self.field_metadata[field]['colnum']
 
         if self.first_sort:
             subsort = True
             self.first_sort = False
-        fcmp = self.seriescmp \
-                if field == 'series' and \
-                   tweaks['title_series_sorting'] == 'library_order' \
-                else \
-                   functools.partial(self.cmp, self.FIELD_MAP[field],
+        if self.field_metadata[field]['is_custom']:
+            if self.field_metadata[field]['datatype'] == 'series':
+                fcmp = functools.partial(self.seriescmp,
+                    self.field_metadata[field]['rec_index'],
+                    self.field_metadata.cc_series_index_column_for(field),
+                    library_order=tweaks['title_series_sorting'] == 'library_order')
+            else:
+                as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
+                field = self.field_metadata[field]['colnum']
+                fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
+                                     subsort=subsort, asstr=as_string)
+        elif field == 'series':
+            fcmp = functools.partial(self.seriescmp, self.FIELD_MAP['series'],
+                self.FIELD_MAP['series_index'],
+                library_order=tweaks['title_series_sorting'] == 'library_order')
+        else:
+            fcmp = functools.partial(self.cmp, self.FIELD_MAP[field],
                                      subsort=subsort, asstr=as_string)
         self._map.sort(cmp=fcmp, reverse=not ascending)
         self._map_filtered = [id for id in self._map if id in self._map_filtered]

src/calibre/library/cli.py

@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO
+import sys, os, cStringIO, re
 from textwrap import TextWrapper
 
 from calibre import terminal_controller, preferred_encoding, prints
-from calibre.utils.config import OptionParser, prefs
+from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
@@ -680,7 +680,29 @@ def command_catalog(args, dbpath):
 
 # end of GR additions
 
+def parse_series_string(db, label, value):
+    val = unicode(value).strip()
+    s_index = None
+    pat = re.compile(r'\[([.0-9]+)\]')
+    match = pat.search(val)
+    if match is not None:
+        val = pat.sub('', val).strip()
+        s_index = float(match.group(1))
+    elif val:
+        if tweaks['series_index_auto_increment'] == 'next':
+            s_index = db.get_next_cc_series_num_for(val, label=label)
+        else:
+            s_index = 1.0
+    return val, s_index
+
 def do_set_custom(db, col, id_, val, append):
+    if db.custom_column_label_map[col]['datatype'] == 'series':
+        val, s_index = parse_series_string(db, col, val)
+        db.set_custom(id_, val, extra=s_index, label=col, append=append)
+        prints('Data set to: %r[%4.2f]'%
+               (db.get_custom(id_, label=col, index_is_id=True),
+                db.get_custom_extra(id_, label=col, index_is_id=True)))
+    else:
         db.set_custom(id_, val, label=col, append=append)
         prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True))
 

src/calibre/library/custom_columns.py

@@ -8,6 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import json
 from functools import partial
+from math import floor
 
 from calibre import prints
 from calibre.constants import preferred_encoding
@@ -16,7 +17,7 @@ from calibre.utils.date import parse_date
 class CustomColumns(object):
 
     CUSTOM_DATA_TYPES = frozenset(['rating', 'text', 'comments', 'datetime',
-        'int', 'float', 'bool'])
+        'int', 'float', 'bool', 'series'])
 
     def custom_table_names(self, num):
         return 'custom_column_%d'%num, 'books_custom_column_%d_link'%num
@@ -137,7 +138,8 @@ class CustomColumns(object):
                 'bool':  adapt_bool,
                 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}),
                 'datetime' : adapt_datetime,
-                'text':adapt_text
+                'text':adapt_text,
+                'series':adapt_text
         }
 
         # Create Tag Browser categories for custom columns
@@ -171,6 +173,19 @@ class CustomColumns(object):
                 ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower()))
         return ans
 
+    def get_custom_extra(self, idx, label=None, num=None, index_is_id=False):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        # add future datatypes with an extra column here
+        if data['datatype'] not in ['series']:
+            return None
+        ign,lt = self.custom_table_names(data['num'])
+        idx = idx if index_is_id else self.id(idx)
+        return self.conn.get('''SELECT extra FROM %s
+                                WHERE book=?'''%lt, (idx,), all=False)
+
     # convenience methods for tag editing
     def get_custom_items_with_ids(self, label=None, num=None):
         if label is not None:
@@ -220,6 +235,28 @@ class CustomColumns(object):
             self.conn.commit()
     # end convenience methods
 
+    def get_next_cc_series_num_for(self, series, label=None, num=None):
+        if label is not None:
+            data = self.custom_column_label_map[label]
+        if num is not None:
+            data = self.custom_column_num_map[num]
+        if data['datatype'] != 'series':
+            return None
+        table, lt = self.custom_table_names(data['num'])
+        # get the id of the row containing the series string
+        series_id = self.conn.get('SELECT id from %s WHERE value=?'%table,
+                                                        (series,), all=False)
+        if series_id is None:
+            return 1.0
+        # get the label of the associated series number table
+        series_num = self.conn.get('''
+                SELECT MAX({lt}.extra) FROM {lt}
+                WHERE {lt}.book IN (SELECT book FROM {lt} where value=?)
+                '''.format(lt=lt), (series_id,), all=False)
+        if series_num is None:
+            return 1.0
+        return floor(series_num+1)
+
     def all_custom(self, label=None, num=None):
         if label is not None:
             data = self.custom_column_label_map[label]
@@ -271,9 +308,8 @@ class CustomColumns(object):
             self.conn.commit()
         return changed
 
-
-
-    def set_custom(self, id_, val, label=None, num=None, append=False, notify=True):
+    def set_custom(self, id_, val, label=None, num=None,
+                   append=False, notify=True, extra=None):
         if label is not None:
             data = self.custom_column_label_map[label]
         if num is not None:
@@ -318,9 +354,16 @@ class CustomColumns(object):
                 if not self.conn.get(
                     'SELECT book FROM %s WHERE book=? AND value=?'%lt,
                                                         (id_, xid), all=False):
+                    if data['datatype'] == 'series':
                         self.conn.execute(
-                        'INSERT INTO %s(book, value) VALUES (?,?)'%lt,
-                            (id_, xid))
+                            '''INSERT INTO %s(book, value, extra)
+                               VALUES (?,?,?)'''%lt, (id_, xid, extra))
+                        self.data.set(id_, self.FIELD_MAP[data['num']]+1,
+                                      extra, row_is_id=True)
+                    else:
+                        self.conn.execute(
+                            '''INSERT INTO %s(book, value)
+                                VALUES (?,?)'''%lt, (id_, xid))
             self.conn.commit()
             nval = self.conn.get(
                     'SELECT custom_%s FROM meta2 WHERE id=?'%data['num'],
@@ -370,6 +413,9 @@ class CustomColumns(object):
                     {table} ON(link.value={table}.id) WHERE link.book=books.id)
                     custom_{num}
                 '''.format(query=query%table, lt=lt, table=table, num=data['num'])
+                if data['datatype'] == 'series':
+                    line += ''',(SELECT extra FROM {lt} WHERE {lt}.book=books.id)
+                        custom_index_{num}'''.format(lt=lt, num=data['num'])
             else:
                 line = '''
                 (SELECT value FROM {table} WHERE book=books.id) custom_{num}
@@ -393,7 +439,7 @@ class CustomColumns(object):
 
         if datatype in ('rating', 'int'):
             dt = 'INT'
-        elif datatype in ('text', 'comments'):
+        elif datatype in ('text', 'comments', 'series'):
             dt = 'TEXT'
         elif datatype in ('float',):
             dt = 'REAL'
@@ -404,6 +450,10 @@ class CustomColumns(object):
         collate = 'COLLATE NOCASE' if dt == 'TEXT' else ''
         table, lt = self.custom_table_names(num)
         if normalized:
+            if datatype == 'series':
+                s_index = 'extra REAL,'
+            else:
+                s_index = ''
             lines = [
                 '''\
                 CREATE TABLE %s(
@@ -419,8 +469,9 @@ class CustomColumns(object):
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     book INTEGER NOT NULL,
                     value INTEGER NOT NULL,
+                    %s
                     UNIQUE(book, value)
-                    );'''%lt,
+                    );'''%(lt, s_index),
 
                 'CREATE INDEX %s_aidx ON %s (value);'%(lt,lt),
                 'CREATE INDEX %s_bidx ON %s (book);'%(lt,lt),

src/calibre/library/database2.py

@@ -237,6 +237,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         self.custom_column_num_map[col]['label'],
                                         base,
                                         prefer_custom=True)
+            if self.custom_column_num_map[col]['datatype'] == 'series':
+                # account for the series index column. Field_metadata knows that
+                # the series index is one larger than the series. If you change
+                # it here, be sure to change it there as well.
+                self.FIELD_MAP[str(col)+'_s_index'] = base = base+1
 
         self.FIELD_MAP['cover'] = base+1
         self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
@@ -777,6 +782,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         icon=icon, tooltip=tooltip)
                                     for r in data if item_not_zero_func(r)]
 
+        # Needed for legacy databases that have multiple ratings that
+        # map to n stars
+        for r in categories['rating']:
+            for x in categories['rating']:
+                if r.name == x.name and r.id != x.id:
+                    r.count = r.count + x.count
+                    categories['rating'].remove(x)
+                    break
+
         # We delayed computing the standard formats category because it does not
         # use a view, but is computed dynamically
         categories['formats'] = []

src/calibre/library/field_metadata.py

@@ -81,7 +81,7 @@ class FieldMetadata(dict):
                            'column':'name',
                            'link_column':'series',
                            'category_sort':'(title_sort(name))',
-                           'datatype':'text',
+                           'datatype':'series',
                            'is_multiple':None,
                            'kind':'field',
                            'name':_('Series'),
@@ -398,6 +398,8 @@ class FieldMetadata(dict):
             if val['is_category'] and val['kind'] in ('user', 'search'):
                 del self._tb_cats[key]
 
+    def cc_series_index_column_for(self, key):
+        return self._tb_cats[key]['rec_index'] + 1
 
     def add_user_category(self, label, name):
         if label in self._tb_cats:

src/calibre/manual/conf.py

@@ -100,9 +100,8 @@ html_use_smartypants = True
 html_title       = 'calibre User Manual'
 html_short_title = 'Start'
 html_logo        = 'resources/logo.png'
-epub_titlepage   = 'resources/titlepage.html'
-epub_logo        = 'resources/logo.png'
 epub_author      = 'Kovid Goyal'
+epub_cover       = 'resources/epub_cover.jpg'
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}

src/calibre/manual/custom.py

@@ -304,9 +304,8 @@ def auto_member(dirname, arguments, options, content, lineno,
     return list(node)
 
 def setup(app):
-    app.add_config_value('epub_titlepage', None, False)
+    app.add_config_value('epub_cover', None, False)
     app.add_config_value('epub_author', '', False)
-    app.add_config_value('epub_logo', None, False)
     app.add_builder(CustomBuilder)
     app.add_builder(CustomQtBuild)
     app.add_builder(EPUBHelpBuilder)

src/calibre/manual/epub.py

@@ -50,6 +50,7 @@ OPF = '''\
         <dc:identifier opf:scheme="sphinx" id="sphinx_id">{uid}</dc:identifier>
         <dc:date>{date}</dc:date>
         <meta name="calibre:publication_type" content="sphinx_manual" />
+        <meta name="cover" content="cover"/>
     </metadata>
     <manifest>
     {manifest}
@@ -71,6 +72,29 @@ CONTAINER='''\
    </rootfiles>
 </container>
 '''
+
+SVG_TEMPLATE = '''\
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+        <meta name="calibre:cover" content="true" />
+        <title>Cover</title>
+        <style type="text/css" title="override_css">
+            @page {padding: 0pt; margin:0pt}
+            body { text-align: center; padding:0pt; margin: 0pt; }
+        </style>
+    </head>
+    <body>
+        <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
+            xmlns:xlink="http://www.w3.org/1999/xlink"
+            width="100%%" height="100%%" viewBox="0 0 600 800"
+            preserveAspectRatio="none">
+            <image width="600" height="800" xlink:href="%s"/>
+        </svg>
+    </body>
+</html>
+'''
+
 class TOC(list):
 
     def __init__(self, title=None, href=None):
@@ -151,8 +175,6 @@ class EPUBHelpBuilder(StandaloneHTMLBuilder):
         spine = [' '*8+'<itemref idref=%s />'%quoteattr(x) for x in self.spine]
         spine = '\n'.join(spine)
         guide = ''
-        if self.conf.epub_titlepage:
-            guide = ' '*8 + '<reference type="cover"  href="_static/titlepage.html" />'
 
         opf = OPF.format(title=escape(self.conf.html_title),
                 author=escape(self.conf.epub_author), uid=str(uuid.uuid4()),
@@ -162,18 +184,15 @@ class EPUBHelpBuilder(StandaloneHTMLBuilder):
         self.manifest['content.opf'] = ('application/oebps-package+xml', 'opf')
 
     def create_titlepage(self):
-        if self.conf.epub_titlepage:
-            img = ''
-            if self.conf.epub_logo:
-                img = '_static/epub_logo'+os.path.splitext(self.conf.epub_logo)[1]
-                shutil.copyfile(self.conf.epub_logo,
-                        os.path.join(self.html_outdir, *img.split('/')))
-            raw = open(self.conf.epub_titlepage, 'rb').read()
-            raw = raw%dict(title=self.conf.html_title,
-                    version=self.conf.version,
-                    img=img.split('/')[-1],
-                    author=self.conf.epub_author)
-            open(os.path.join(self.html_outdir, '_static', 'titlepage.html'), 'wb').write(raw)
+        self.cover_image_url = None
+        if self.conf.epub_cover:
+            img = '_static/'+os.path.basename(self.conf.epub_cover)
+            shutil.copyfile(self.conf.epub_cover, os.path.join(self.html_outdir,
+                *img.split('/')))
+            self.cover_image_url = img
+            tp = SVG_TEMPLATE%img.split('/')[-1]
+            open(os.path.join(self.html_outdir, '_static', 'titlepage.html'),
+                    'wb').write(tp)
 
     def generate_manifest(self):
         self.manifest = {}
@@ -190,8 +209,12 @@ class EPUBHelpBuilder(StandaloneHTMLBuilder):
                     self.manifest[url] = 'application/octet-stream'
                 if self.manifest[url] == 'text/html':
                     self.manifest[url] = 'application/xhtml+xml'
-                self.manifest[url] = (self.manifest[url], 'id'+str(id))
+                if self.cover_image_url and url.endswith(self.cover_image_url):
+                    id_ = 'cover'
+                else:
+                    id_ = 'id'+str(id)
                     id += 1
+                self.manifest[url] = (self.manifest[url], id_)
 
     def isdocnode(self, node):
         if not isinstance(node, nodes.list_item):
@@ -227,7 +250,7 @@ class EPUBHelpBuilder(StandaloneHTMLBuilder):
         open('toc.ncx', 'wb').write(ncx)
         self.manifest['toc.ncx'] = ('application/x-dtbncx+xml', 'ncx')
         self.spine.insert(0, self.manifest[self.conf.master_doc+'.html'][1])
-        if self.conf.epub_titlepage:
+        if self.conf.epub_cover:
             self.spine.insert(0, self.manifest['_static/titlepage.html'][1])
 
     def add_to_spine(self, href):

src/calibre/manual/faq.rst

@@ -101,6 +101,33 @@ We just need some information from you:
 Once you send us the output for a particular operating system, support for the device in that operating system
 will appear in the next release of |app|.
 
+How does |app| manage collections on my SONY reader?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When |app| connects with the device, it retrieves all collections for the books on the device. The collections
+of which books are members are shown on the device view.
+
+When you send a book to the device, |app| will add the book to collections based on the metadata for that book. By
+default, collections are created from tags and series. You can control what metadata is used by going to
+Preferences->Plugins->Device Interface plugins and customizing the SONY device interface plugin. If you remove all
+values, |app| will not add the book to any collection.
+
+Collection management is largely controlled by 'Preserve device collections' found at Preferences->Add/Save->Sending
+to device. If checked (the default), managing collections is left to the user; |app| will not delete already
+existing collections for a book on your device when you resend the book to the device, but |app| will add the book to
+collections if necessary.  To ensure that the collections for a book are based only on current |app| metadata, first
+delete the books from the device, then resend the books.  You can edit collections directly on the device view by
+double-clicking or right-clicking in the collections column.
+
+If 'Preserve device collections' is not checked, then |app| will manage collections. Collections will be built using
+|app| metadata exclusively.  Sending a book to the device will correct the collections for that book so its
+collections exactly match the book's metadata. Collections are added and deleted as necessary.  Editing collections on
+the device pane is not permitted, because collections not in the metadata will be removed automatically.
+
+In summary, check 'Preserve device collections' if you want to manage collections yourself.  Collections for a book
+will never be removed by |app|, but can be removed by you by editing on the device view.  Uncheck 'Preserve device
+collections' if you want |app| to manage the collections, adding books to and removing books from collections as
+needed.  
 
 Can I use both |app| and the SONY software to manage my reader?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -160,7 +187,7 @@ Alternative for the iPad
 As of |app| version 0.7.0, you can plugin your iPad into the computer using its charging cable, and |app| will detect it and show you a list of books on the iPad. You can then use the Send to device button to send books directly to iBooks on the iPad.
 
 This method only works on Windows XP and higher and OS X 10.5 and higher. Linux is not supported (iTunes is not available in linux) and OS X 10.4 is not supported. For more details, see
-`this forum post http://www.mobileread.com/forums/showpost.php?p=944079&postcount=1`_.
+`this forum post <http://www.mobileread.com/forums/showpost.php?p=944079&postcount=1>`_.
 
 How do I use |app| with my Android phone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -190,11 +217,16 @@ The most likely cause of this is your antivirus program. Try temporarily disabli
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-|app| uses something called SYSFS to detect devices in linux. The linux kernel can export two version of SYSFS, one of which is deprecated. Some linux distributions still ship with kernels that support the deprecated version of SYSFS, even though it was deprecated a long time ago. In this case, device detection in |app| will not work. You can check what version of SYSFS is exported by your kernel with the following command::
+|app| needs your linux kernel to have been setup correctly to detect devices. If your devices are not detected, perform the following tests::
 
     grep SYSFS_DEPRECATED /boot/config-`uname -r`
 
-You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``. If you don't you have to either recompile your kernel with the correct setting, or upgrade your linux distro to a more modern version, where this will not be set.
+You should see something like ``CONFIG_SYSFS_DEPRECATED_V2 is not set``.
+Also, ::
+
+    grep CONFIG_SCSI_MULTI_LUN /boot/config-`uname -r`
+
+must return ``CONFIG_SCSI_MULTI_LUN=y``. If you don't see either, you have to recompile your kernel with the correct settings.
 
 Library Management
 ------------------
@@ -280,7 +312,7 @@ Take your pick:
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts founf on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory. 
+|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts found on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
 
 |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -303,6 +335,10 @@ Post any output you see in a help message on the `Forum <http://www.mobileread.c
 |app| is not starting on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+One common cause of failures on OS X is the use of accessibility technologies that are incompatible with the graphics toolkit |app| uses.
+Try turning off VoiceOver if you have it on. Also go to System Preferences->System->Universal Access and turn off the setting for enabling
+access for assistive devices in all the tabs.
+
 You can obtain debug output about why |app| is not starting by running `Console.app`. Debug output will
 be printed to it. If the debug output contains a line that looks like::
 
@@ -312,9 +348,9 @@ then the problem is probably a corrupted font cache. You can clear the cache by
 `instructions <http://www.macworld.com/article/139383/2009/03/fontcacheclear.html>`_. If that doesn't
 solve it, look for a corrupted font file on your system, in ~/Library/Fonts or the like.
 
-
 My antivirus program claims |app| is a virus/trojan?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Your antivirus program is wrong. |app| is a completely open source product. You can actually browse the source code yourself (or hire someone to do it for you) to verify that it is not a virus. Please report the false identification to whatever company you buy your antivirus software from. If the antivirus program is preventing you from downloading/installing |app|, disable it temporarily, install |app| and then re-enable it.
 
 How do I use purchased EPUB books with |app|?

src/calibre/manual/resources/titlepage.html

@@ -1,29 +0,0 @@
-<html  xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
-    <head>
-        <title>%(title)s</title>
-        <style type="text/css">
-            body {
-                text-align: center;
-                vertical-align: center;
-                overflow: hidden;
-                font-size: 16pt;
-            }
-            .logo {
-                text-align:center;
-                font-size: 1pt;
-                overflow:hidden;
-            }
-            h1 { font-family: serif; }
-            h2, h4 { font-family: monospace; }
-        </style>
-    </head>
-    <body>
-        <h1>%(title)s</h1>
-        <h4 style="font-family:monospace">%(version)s</h4>
-        <div style="text-align:center">
-            <img class="logo" src="%(img)s" alt="calibre logo" />
-        </div>
-        <h2>%(author)s</h2>
-    </body>
-</html>
-

src/calibre/utils/config.py

@@ -698,6 +698,8 @@ def _prefs():
     # calibre server can execute searches
     c.add_opt('saved_searches', default={}, help=_('List of named saved searches'))
     c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
+    c.add_opt('preserve_user_collections', default=True,
+        help=_('Preserve all collections even if not in library metadata.'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c

src/calibre/web/feeds/news.py

@@ -585,6 +585,8 @@ class BasicNewsRecipe(Recipe):
         self.lrf = options.lrf
         self.output_profile = options.output_profile
         self.touchscreen = getattr(self.output_profile, 'touchscreen', False)
+        if self.touchscreen:
+            self.template_css += self.output_profile.touchscreen_news_css
 
         self.output_dir = os.path.abspath(self.output_dir)
         if options.test:
@@ -638,7 +640,8 @@ class BasicNewsRecipe(Recipe):
         if self.delay > 0:
             self.simultaneous_downloads = 1
 
-        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else templates.NavBarTemplate()
+        self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
+                      templates.NavBarTemplate()
         self.failed_downloads = []
         self.partial_failures = []
 
@@ -726,7 +729,6 @@ class BasicNewsRecipe(Recipe):
         timefmt = self.timefmt
         if self.touchscreen:
             templ = templates.TouchscreenIndexTemplate()
-            timefmt = '%A, %d %b %Y'
         return templ.generate(self.title, "mastheadImage.jpg", timefmt, feeds,
                               extra_css=css).render(doctype='xhtml')
 
@@ -752,7 +754,8 @@ class BasicNewsRecipe(Recipe):
 
 
 
-    def feed2index(self, feed):
+    def feed2index(self, f, feeds):
+        feed = feeds[f]
         if feed.image_url is not None: # Download feed image
             imgdir = os.path.join(self.output_dir, 'images')
             if not os.path.isdir(imgdir):
@@ -782,33 +785,9 @@ class BasicNewsRecipe(Recipe):
         css = self.template_css + '\n\n' +(self.extra_css if self.extra_css else '')
 
         if self.touchscreen:
-            touchscreen_css = u'''
-                    .summary_headline {
-                        font-weight:bold; text-align:left;
-                    }
-
-                    .summary_byline {
-                        text-align:left;
-                        font-family:monospace;
-                    }
-
-                    .summary_text {
-                        text-align:left;
-                    }
-
-                    .feed {
-                        font-family:sans-serif; font-weight:bold; font-size:larger;
-                    }
-
-                    .calibre_navbar {
-                        font-family:monospace;
-                    }
-
-            '''
-
             templ = templates.TouchscreenFeedTemplate()
-            css = touchscreen_css + '\n\n' + (self.extra_css if self.extra_css else '')
-        return templ.generate(feed, self.description_limiter,
+
+        return templ.generate(f, feeds, self.description_limiter,
                               extra_css=css).render(doctype='xhtml')
 
 
@@ -951,7 +930,7 @@ class BasicNewsRecipe(Recipe):
         #feeds.restore_duplicates()
 
         for f, feed in enumerate(feeds):
-            html = self.feed2index(feed)
+            html = self.feed2index(f,feeds)
             feed_dir = os.path.join(self.output_dir, 'feed_%d'%f)
             with open(os.path.join(feed_dir, 'index.html'), 'wb') as fi:
                 fi.write(html)

src/calibre/web/feeds/templates.py

@@ -3,9 +3,12 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
+
+import copy
+
 from lxml import html, etree
 from lxml.html.builder import HTML, HEAD, TITLE, STYLE, DIV, BODY, \
-        STRONG, BR, SPAN, A, HR, UL, LI, H2, IMG, P as PT, \
+        STRONG, EM, BR, SPAN, A, HR, UL, LI, H2, H3, IMG, P as PT, \
         TABLE, TD, TR
 
 from calibre import preferred_encoding, strftime, isbytestring
@@ -14,6 +17,7 @@ def CLASS(*args, **kwargs): # class is a reserved word in Python
     kwargs['class'] = ' '.join(args)
     return kwargs
 
+# Regular templates
 class Template(object):
 
     IS_HTML = True
@@ -44,105 +48,35 @@ class Template(object):
         return etree.tostring(self.root, encoding='utf-8', xml_declaration=True,
                 pretty_print=True)
 
-class NavBarTemplate(Template):
+class EmbeddedContent(Template):
 
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-                 two_levels, url, __appname__, prefix='', center=True,
-                 extra_css=None, style=None):
-        head = HEAD(TITLE('navbar'))
+    def _generate(self, article, style=None, extra_css=None):
+        content = article.content if article.content else ''
+        summary = article.summary if article.summary else ''
+        text = content if len(content) > len(summary) else summary
+        head = HEAD(TITLE(article.title))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if prefix and not prefix.endswith('/'):
-            prefix += '/'
-        align = 'center' if center else 'left'
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
-            style='text-align:'+align))
-        if bottom:
-            navbar.append(HR())
-            text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
-            p[0].tail = ' from '
-            navbar.append(p)
-            navbar.append(BR())
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(HR())
-
-        self.root = HTML(head, BODY(navbar))
-
-class TouchscreenNavBarTemplate(Template):
-
-    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
-                 two_levels, url, __appname__, prefix='', center=True,
-                 extra_css=None, style=None):
-        head = HEAD(TITLE('navbar'))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        if prefix and not prefix.endswith('/'):
-            prefix += '/'
-        align = 'center' if center else 'left'
-        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_100',
-            style='text-align:'+align))
-        if bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-            text = 'This article was downloaded by '
-            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
-            p[0].tail = ' from '
-            navbar.append(p)
-            navbar.append(BR())
-            navbar.append(BR())
-        else:
-            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
-                    else 'article_%d'%(art+1)
-            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
-            href = '%s%s/%s/index.html'%(prefix, up, next)
-            navbar.text = '| '
-            navbar.append(A('Next', href=href))
-
-        href = '%s../index.html#article_%d'%(prefix, art)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Section Menu', href=href))
-        href = '%s../../index.html#feed_%d'%(prefix, feed)
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        navbar.append(A('Main Menu', href=href))
-        if art > 0 and not bottom:
-            href = '%s../article_%d/index.html'%(prefix, art-1)
-            navbar.iterchildren(reversed=True).next().tail = ' | '
-            navbar.append(A('Previous', href=href))
-
-        navbar.iterchildren(reversed=True).next().tail = ' | '
-        if not bottom:
-            navbar.append(DIV(style="border-top:1px solid gray;border-bottom:1em solid white"))
-
-        self.root = HTML(head, BODY(navbar))
+        if isbytestring(text):
+            text = text.decode('utf-8', 'replace')
+        elements = html.fragments_fromstring(text)
+        self.root = HTML(head,
+                BODY(H2(article.title), DIV()))
+        div = self.root.find('body').find('div')
+        if elements and isinstance(elements[0], unicode):
+            div.text = elements[0]
+            elements = list(elements)[1:]
+        for elem in elements:
+            elem.getparent().remove(elem)
+            div.append(elem)
 
 class IndexTemplate(Template):
 
     def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+        self.IS_HTML = False
         if isinstance(datefmt, unicode):
             datefmt = datefmt.encode(preferred_encoding)
         date = strftime(datefmt)
@@ -164,43 +98,10 @@ class IndexTemplate(Template):
                 CLASS('calibre_rescale_100'))
         self.root = HTML(head, BODY(div))
 
-class TouchscreenIndexTemplate(Template):
-
-    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
-        if isinstance(datefmt, unicode):
-            datefmt = datefmt.encode(preferred_encoding)
-        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
-        masthead_p = etree.Element("p")
-        masthead_p.set("style","text-align:center")
-        masthead_img = etree.Element("img")
-        masthead_img.set("src",masthead)
-        masthead_img.set("alt","masthead")
-        masthead_p.append(masthead_img)
-
-        head = HEAD(TITLE(title))
-        if style:
-            head.append(STYLE(style, type='text/css'))
-        if extra_css:
-            head.append(STYLE(extra_css, type='text/css'))
-
-        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
-        for i, feed in enumerate(feeds):
-            if feed:
-                tr = TR()
-                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
-                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
-                toc.append(tr)
-        div = DIV(
-                masthead_p,
-                PT(date, style='text-align:center'),
-                #DIV(style="border-color:gray;border-top-style:solid;border-width:thin"),
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white"),
-                toc)
-        self.root = HTML(head, BODY(div))
-
 class FeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+        feed = feeds[f]
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -248,9 +149,146 @@ class FeedTemplate(Template):
 
         self.root = HTML(head, body)
 
+class NavBarTemplate(Template):
+
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None, style=None):
+        head = HEAD(TITLE('navbar'))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        if prefix and not prefix.endswith('/'):
+            prefix += '/'
+        align = 'center' if center else 'left'
+
+        navbar = DIV(CLASS('calibre_navbar', 'calibre_rescale_70',
+            style='text-align:'+align))
+        if bottom:
+            navbar.append(HR())
+            text = 'This article was downloaded by '
+            p = PT(text, STRONG(__appname__), A(url, href=url), style='text-align:left')
+            p[0].tail = ' from '
+            navbar.append(p)
+            navbar.append(BR())
+            navbar.append(BR())
+        else:
+            next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                    else 'article_%d'%(art+1)
+            up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+            href = '%s%s/%s/index.html'%(prefix, up, next)
+            navbar.text = '| '
+            navbar.append(A('Next', href=href))
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Section Menu', href=href))
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        navbar.append(A('Main Menu', href=href))
+        if art > 0 and not bottom:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar.iterchildren(reversed=True).next().tail = ' | '
+            navbar.append(A('Previous', href=href))
+        navbar.iterchildren(reversed=True).next().tail = ' | '
+        if not bottom:
+            navbar.append(HR())
+
+        self.root = HTML(head, BODY(navbar))
+
+
+# Touchscreen templates
+class TouchscreenIndexTemplate(Template):
+
+    def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
+
+        self.IS_HTML = False
+
+        if isinstance(datefmt, unicode):
+            datefmt = datefmt.encode(preferred_encoding)
+        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
+        masthead_p = etree.Element("p")
+        masthead_p.set("style","text-align:center")
+        masthead_img = etree.Element("img")
+        masthead_img.set("src",masthead)
+        masthead_img.set("alt","masthead")
+        masthead_p.append(masthead_img)
+
+        head = HEAD(TITLE(title))
+        if style:
+            head.append(STYLE(style, type='text/css'))
+        if extra_css:
+            head.append(STYLE(extra_css, type='text/css'))
+
+        toc = TABLE(CLASS('toc'),width="100%",border="0",cellpadding="3px")
+        for i, feed in enumerate(feeds):
+            if feed:
+                tr = TR()
+                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
+                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
+                toc.append(tr)
+        div = DIV(
+                masthead_p,
+                H3(CLASS('publish_date'),date),
+                DIV(CLASS('divider')),
+                toc)
+        self.root = HTML(head, BODY(div))
+
 class TouchscreenFeedTemplate(Template):
 
-    def _generate(self, feed, cutoff, extra_css=None, style=None):
+    def _generate(self, f, feeds, cutoff, extra_css=None, style=None):
+
+        def trim_title(title,clip=18):
+            if len(title)>clip:
+                tokens = title.split(' ')
+                new_title_tokens = []
+                new_title_len = 0
+                if len(tokens[0]) > clip:
+                    return tokens[0][:clip] + '...'
+                for token in tokens:
+                    if len(token) + new_title_len < clip:
+                        new_title_tokens.append(token)
+                        new_title_len += len(token)
+                    else:
+                        new_title_tokens.append('...')
+                        title = ' '.join(new_title_tokens)
+                        break
+            return title
+
+        self.IS_HTML = False
+        feed = feeds[f]
+
+        # Construct the navbar
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
+        navbar_tr = TR()
+
+        # Previous Section
+        link = ''
+        if f > 0:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f-1].title),
+                     href = '../feed_%d/index.html' % int(f-1))
+        navbar_tr.append(TD(link, width="40%", align="center"))
+
+        # Up to Sections
+        link = A(STRONG('Sections'), href="../index.html")
+        navbar_tr.append(TD(link,width="20%",align="center"))
+
+        # Next Section
+        link = ''
+        if f < len(feeds)-1:
+            link = A(CLASS('feed_link'),
+                     trim_title(feeds[f+1].title),
+                     href = '../feed_%d/index.html' % int(f+1))
+        navbar_tr.append(TD(link, width="40%", align="center", ))
+        navbar_t.append(navbar_tr)
+        top_navbar = navbar_t
+        bottom_navbar = copy.copy(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar_t, pretty_print=True)
+
+
+        # Build the page
         head = HEAD(TITLE(feed.title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -258,10 +296,11 @@ class TouchscreenFeedTemplate(Template):
             head.append(STYLE(extra_css, type='text/css'))
         body = BODY(style='page-break-before:always')
         div = DIV(
-                H2(feed.title, CLASS('calibre_feed_title', 'calibre_rescale_160')),
-                DIV(style="border-top:1px solid gray;border-bottom:1em solid white")
+                top_navbar,
+                H2(feed.title, CLASS('feed_title'))
                 )
         body.append(div)
+
         if getattr(feed, 'image', None):
             div.append(DIV(IMG(
                 alt = feed.image_alt if feed.image_alt else '',
@@ -280,7 +319,6 @@ class TouchscreenFeedTemplate(Template):
                 continue
             tr = TR()
 
-            if True:
             div_td = DIV(
                     A(article.title, CLASS('summary_headline','calibre_rescale_120',
                                     href=article.url)),
@@ -292,53 +330,53 @@ class TouchscreenFeedTemplate(Template):
                 div_td.append(DIV(cutoff(article.text_summary),
                     CLASS('summary_text', 'calibre_rescale_100')))
             tr.append(TD(div_td))
-            else:
-                td = TD(
-                        A(article.title, CLASS('summary_headline','calibre_rescale_120',
-                                        href=article.url))
-                        )
-                if article.author:
-                    td.append(DIV(article.author,
-                        CLASS('summary_byline', 'calibre_rescale_100')))
-                if article.summary:
-                    td.append(DIV(cutoff(article.text_summary),
-                        CLASS('summary_text', 'calibre_rescale_100')))
-
-                tr.append(td)
-
             toc.append(tr)
+
         div.append(toc)
-
-        navbar = DIV('| ', CLASS('calibre_navbar', 'calibre_rescale_100'),style='text-align:center')
-        link = A('Up one level', href="../index.html")
-        link.tail = ' |'
-        navbar.append(link)
-        div.append(navbar)
-
+        div.append(BR())
+        div.append(bottom_navbar)
         self.root = HTML(head, body)
 
-class EmbeddedContent(Template):
+class TouchscreenNavBarTemplate(Template):
 
-    def _generate(self, article, style=None, extra_css=None):
-        content = article.content if article.content else ''
-        summary = article.summary if article.summary else ''
-        text = content if len(content) > len(summary) else summary
-        head = HEAD(TITLE(article.title))
+    def _generate(self, bottom, feed, art, number_of_articles_in_feed,
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None, style=None):
+        head = HEAD(TITLE('navbar'))
         if style:
             head.append(STYLE(style, type='text/css'))
         if extra_css:
             head.append(STYLE(extra_css, type='text/css'))
 
-        if isbytestring(text):
-            text = text.decode('utf-8', 'replace')
-        elements = html.fragments_fromstring(text)
-        self.root = HTML(head,
-                BODY(H2(article.title), DIV()))
-        div = self.root.find('body').find('div')
-        if elements and isinstance(elements[0], unicode):
-            div.text = elements[0]
-            elements = list(elements)[1:]
-        for elem in elements:
-            elem.getparent().remove(elem)
-            div.append(elem)
+        navbar = DIV()
+        navbar_t = TABLE(CLASS('touchscreen_navbar'))
+        navbar_tr = TR()
 
+        # | Previous
+        if art > 0:
+            href = '%s../article_%d/index.html'%(prefix, art-1)
+            navbar_tr.append(TD(A(EM('Previous'),href=href),
+                               width="32%"))
+        else:
+            navbar_tr.append(TD('', width="32%"))
+
+        # | Articles | Sections |
+        href = '%s../index.html#article_%d'%(prefix, art)
+        navbar_tr.append(TD(A(STRONG('Articles'), href=href),width="18%"))
+
+        href = '%s../../index.html#feed_%d'%(prefix, feed)
+        navbar_tr.append(TD(A(STRONG('Sections'), href=href),width="18%"))
+
+        # | Next
+        next = 'feed_%d'%(feed+1) if art == number_of_articles_in_feed - 1 \
+                else 'article_%d'%(art+1)
+        up = '../..' if art == number_of_articles_in_feed - 1 else '..'
+        href = '%s%s/%s/index.html'%(prefix, up, next)
+
+        navbar_tr.append(TD(A(EM('Next'),href=href),
+                           width="32%"))
+        navbar_t.append(navbar_tr)
+        navbar.append(navbar_t)
+        #print "\n%s\n" % etree.tostring(navbar, pretty_print=True)
+
+        self.root = HTML(head, BODY(navbar))