Merge from source

resources/content_server/browse/browse.js

@@ -156,6 +156,7 @@ function category() {
                 if (href) {
                     $.ajax({
                         url:href,
+                        cache: false,
                         data:{'sort':cookie(sort_cookie_name)},
                         success: function(data) {
                             this.children(".loaded").html(data);
@@ -212,6 +213,7 @@ function load_page(elem) {
         url: href,
         context: elem,
         dataType: "json",
+        cache : false,
         type: 'POST',
         timeout: 600000, //milliseconds (10 minutes)
         data: {'ids': ids},
@@ -263,6 +265,7 @@ function show_details(a_dom) {
     $.ajax({
         url: book.find('.details-href').attr('title'),
         context: bd,
+        cache: false,
         dataType: "json",
         timeout: 600000, //milliseconds (10 minutes)
         error: function(xhr, stat, err) {

resources/content_server/mobile.css

@@ -1,4 +1,10 @@
 /* CSS for the mobile version of the content server webpage */
+
+
+.body {
+    font-family: sans-serif;
+}
+
 .navigation table.buttons {
 	width: 100%;
 }
@@ -79,5 +85,20 @@ div.navigation {
 }
 
 #spacer {
-	clear: both;
+    clear: both;
-}
+}
+
+.data-container {
+    display: inline-block;
+    vertical-align: middle;
+}
+
+.first-line {
+    font-size: larger;
+    font-weight: bold;
+}
+
+.second-line {
+    margin-top: 0.75ex;
+    display: block;
+}

resources/default_tweaks.py

@@ -106,7 +106,8 @@ title_sort_articles=r'^(A|The|An)\s+'
 auto_connect_to_folder = ''
 
 
-# Specify renaming rules for sony collections. Collections on Sonys are named
+# Specify renaming rules for sony collections. This tweak is only applicable if
+# metadata management is set to automatic. Collections on Sonys are named
 # depending upon whether the field is standard or custom. A collection derived
 # from a standard field is named for the value in that field. For example, if
 # the standard 'series' column contains the name 'Darkover', then the series
@@ -137,6 +138,24 @@ auto_connect_to_folder = ''
 sony_collection_renaming_rules={}
 
 
+# Specify how sony collections are sorted. This tweak is only applicable if
+# metadata management is set to automatic. You can indicate which metadata is to
+# be used to sort on a collection-by-collection basis. The format of the tweak
+# is a list of metadata fields from which collections are made, followed by the
+# name of the metadata field containing the sort value.
+# Example: The following indicates that collections built from pubdate and tags
+# are to be sorted by the value in the custom column '#mydate', that collections
+# built from 'series' are to be sorted by 'series_index', and that all other
+# collections are to be sorted by title. If a collection metadata field is not
+# named, then if it is a series- based collection it is sorted by series order,
+# otherwise it is sorted by title order.
+# [(['pubdate', 'tags'],'#mydate'), (['series'],'series_index'), (['*'], 'title')]
+# Note that the bracketing and parentheses are required. The syntax is
+# [ ( [list of fields], sort field ) , ( [ list of fields ] , sort field ) ]
+# Default: empty (no rules), so no collection attributes are named.
+sony_collection_sorting_rules = []
+
+
 # Create search terms to apply a query across several built-in search terms.
 # Syntax: {'new term':['existing term 1', 'term 2', ...], 'new':['old'...] ...}
 # Example: create the term 'myseries' that when used as myseries:foo would

resources/recipes/atlantic.recipe

@@ -71,7 +71,9 @@ class TheAtlantic(BasicNewsRecipe):
         for poem in soup.findAll('div', attrs={'class':'poem'}):
             title = self.tag_to_string(poem.find('h4'))
             desc  = self.tag_to_string(poem.find(attrs={'class':'author'}))
-            url   = 'http://www.theatlantic.com'+poem.find('a')['href']
+            url   = poem.find('a')['href']
+            if url.startswith('/'):
+                url = 'http://www.theatlantic.com' + url
             self.log('\tFound article:', title, 'at', url)
             self.log('\t\t', desc)
             poems.append({'title':title, 'url':url, 'description':desc,
@@ -83,7 +85,9 @@ class TheAtlantic(BasicNewsRecipe):
         if div is not None:
             self.log('Found section: Advice')
             title = self.tag_to_string(div.find('h4'))
-            url = 'http://www.theatlantic.com'+div.find('a')['href']
+            url = div.find('a')['href']
+            if url.startswith('/'):
+                url = 'http://www.theatlantic.com' + url
             desc = self.tag_to_string(div.find('p'))
             self.log('\tFound article:', title, 'at', url)
             self.log('\t\t', desc)

resources/recipes/cacm.recipe

@@ -1,37 +1,37 @@
-import datetime
+import datetime
-from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.web.feeds.news import BasicNewsRecipe
-
+
-class AdvancedUserRecipe1286242553(BasicNewsRecipe):
+class AdvancedUserRecipe1286242553(BasicNewsRecipe):
-    title          = u'CACM'
+    title          = u'CACM'
-    oldest_article = 7
+    oldest_article = 7
-    max_articles_per_feed = 100
+    max_articles_per_feed = 100
-    needs_subscription = True
+    needs_subscription = True
-    feeds          = [(u'CACM', u'http://cacm.acm.org/magazine.rss')]
+    feeds          = [(u'CACM', u'http://cacm.acm.org/magazine.rss')]
-    language = 'en'
+    language = 'en'
-    __author__ = 'jonmisurda'
+    __author__ = 'jonmisurda'
-    no_stylesheets        = True
+    no_stylesheets        = True
-    remove_tags = [
+    remove_tags = [
-        dict(name='div', attrs={'class':['FeatureBox', 'ArticleComments', 'SideColumn', \
+        dict(name='div', attrs={'class':['FeatureBox', 'ArticleComments', 'SideColumn', \
-              'LeftColumn', 'RightColumn', 'SiteSearch', 'MainNavBar','more', 'SubMenu', 'inner']})
+              'LeftColumn', 'RightColumn', 'SiteSearch', 'MainNavBar','more', 'SubMenu', 'inner']})
-    ]
+    ]
-    cover_url_pattern = 'http://cacm.acm.org/magazines/%d/%d'
+    cover_url_pattern = 'http://cacm.acm.org/magazines/%d/%d'
-
+
-    def get_browser(self):
+    def get_browser(self):
-        br = BasicNewsRecipe.get_browser()
+        br = BasicNewsRecipe.get_browser()
-        if self.username is not None and self.password is not None:
+        if self.username is not None and self.password is not None:
-            br.open('https://cacm.acm.org/login')
+            br.open('https://cacm.acm.org/login')
-            br.select_form(nr=1)
+            br.select_form(nr=1)
-            br['current_member[user]']   = self.username
+            br['current_member[user]']   = self.username
-            br['current_member[passwd]'] = self.password
+            br['current_member[passwd]'] = self.password
-            br.submit()
+            br.submit()
-        return br
+        return br
-
+
-    def get_cover_url(self):
+    def get_cover_url(self):
-        now = datetime.datetime.now()
+        now = datetime.datetime.now()
-
+
-        cover_url = None
+        cover_url = None
-        soup = self.index_to_soup(self.cover_url_pattern % (now.year, now.month))
+        soup = self.index_to_soup(self.cover_url_pattern % (now.year, now.month))
-        cover_item = soup.find('img',attrs={'alt':'magazine cover image'})
+        cover_item = soup.find('img',attrs={'alt':'magazine cover image'})
-        if cover_item:
+        if cover_item:
-           cover_url = cover_item['src']
+           cover_url = cover_item['src']
-        return cover_url
+        return cover_url

resources/recipes/el_pais.recipe

@@ -2,7 +2,7 @@
 __license__   = 'GPL v3'
 __author__    = 'Jordi Balcells, based on an earlier version by Lorenzo Vigentini & Kovid Goyal'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-description   = 'Main daily newspaper from Spain - v1.03 (03, September 2010)'
+description   = 'Main daily newspaper from Spain - v1.04 (19, October 2010)'
 __docformat__ = 'restructuredtext en'
 
 '''
@@ -32,19 +32,16 @@ class ElPais(BasicNewsRecipe):
     remove_javascript = True
     no_stylesheets = True
 
-    keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia','cabecera_noticia_reportaje','cabecera_noticia_opinion','contenido_noticia','caja_despiece','presentacion']})]
+    keep_only_tags = [ dict(name='div', attrs={'class':['cabecera_noticia_reportaje estirar','cabecera_noticia_opinion estirar','cabecera_noticia estirar','contenido_noticia','caja_despiece']})]
-	
-    extra_css      = '''
-                        p{style:normal size:12 serif}
 
-                    '''
+    extra_css             = ' p{text-align: justify; font-size: 100%} body{ text-align: left; font-family: serif; font-size: 100% } h1{ font-family: sans-serif; font-size:200%; font-weight: bolder; text-align: justify; } h2{ font-family: sans-serif; font-size:150%; font-weight: 500; text-align: justify } h3{ font-family: sans-serif; font-size:125%; font-weight: 500; text-align: justify } img{margin-bottom: 0.4em} '
 
     remove_tags    = [
                         dict(name='div', attrs={'class':['zona_superior','pie_enlaces_inferiores','contorno_f','ampliar']}),
-                        dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}),
+                        dict(name='div', attrs={'class':['limpiar','mod_apoyo','borde_sup','votos estirar','info_complementa','info_relacionada','buscador_m','nav_ant_sig']}),
                         dict(name='div', attrs={'id':['suscribirse suscrito','google_noticia','utilidades','coment','foros_not','pie','lomas','calendar']}),
                         dict(name='p', attrs={'class':'nav_meses'}),
-                        dict(attrs={'class':['enlaces_m','miniaturas_m']})
+                        dict(attrs={'class':['enlaces_m','miniaturas_m','nav_miniaturas_m']})
                     ]
 
     feeds          = [

resources/recipes/foxnews.recipe

@@ -4,7 +4,6 @@ __copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
 foxnews.com
 '''
 
-import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class FoxNews(BasicNewsRecipe):
@@ -21,11 +20,10 @@ class FoxNews(BasicNewsRecipe):
     language              = 'en'
     publication_type      = 'newsportal'
     remove_empty_feeds    = True
-    extra_css             = ' body{font-family: Arial,sans-serif } img{margin-bottom: 0.4em} .caption{font-size: x-small} '
+    extra_css             = """
-
+                                body{font-family: Arial,sans-serif }
-    preprocess_regexps = [
+                                .caption{font-size: x-small}
-       (re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')
+                            """
-    ]
 
     conversion_options = {
                           'comment'   : description
@@ -34,27 +32,15 @@ class FoxNews(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes = ['xmlns']
+    remove_attributes = ['xmlns','lang']
-
-    keep_only_tags      = [
-                            dict(name='div', attrs={'id'   :['story','browse-story-content']})
-                           ,dict(name='div', attrs={'class':['posts articles','slideshow']})
-                           ,dict(name='h4' , attrs={'class':'storyDate'})
-                           ,dict(name='h1' , attrs={'xmlns:functx':'http://www.functx.com'})
-                           ,dict(name='div', attrs={'class':'authInfo'})
-                           ,dict(name='div', attrs={'id':'articleCont'})
-                          ]
 
     remove_tags = [
-                     dict(name='div', attrs={'class':['share-links','quigo quigo2','share-text','storyControls','socShare','btm-links']})
+                     dict(name=['object','embed','link','script','iframe','meta','base'])
-                    ,dict(name='div', attrs={'id'   :['otherMedia','loomia_display','img-all-path','story-vcmId','story-url','pane-browse-story-comments','story_related']})
+                    ,dict(attrs={'class':['user-control','url-description','ad-context']})
-                    ,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2','tabs']})
-                    ,dict(name='a' , attrs={'class':'join-discussion'})
-                    ,dict(name='ul' , attrs={'class':['tools','tools alt','tools alt2']})
-                    ,dict(name='p' , attrs={'class':'see_fullarchive'})
-                    ,dict(name=['object','embed','link','script'])
                   ]
 
+    remove_tags_before=dict(name='h1')
+    remove_tags_after =dict(attrs={'class':'url-description'})
 
     feeds = [
               (u'Latest Headlines', u'http://feeds.foxnews.com/foxnews/latest'        )
@@ -67,8 +53,5 @@ class FoxNews(BasicNewsRecipe):
              ,(u'Entertainment'   , u'http://feeds.foxnews.com/foxnews/entertainment' )
             ]
 
-    def preprocess_html(self, soup):
+    def print_version(self, url):
-        for item in soup.findAll(style=True):
+        return url + 'print'
-            del item['style']
-        return self.adeify_images(soup)
-

resources/recipes/new_scientist.recipe

@@ -8,11 +8,11 @@ import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
-    title                 = 'New Scientist - Online News'
+    title                 = 'New Scientist - Online News w. subscription'
     __author__            = 'Darko Miletic'
     description           = 'Science news and science articles from New Scientist.'
     language              = 'en'
-    publisher             = 'New Scientist'
+    publisher             = 'Reed Business Information Ltd.'
     category              = 'science news, science articles, science jobs, drugs, cancer, depression, computer software'
     oldest_article        = 7
     max_articles_per_feed = 100
@@ -21,7 +21,12 @@ class NewScientist(BasicNewsRecipe):
     cover_url             = 'http://www.newscientist.com/currentcover.jpg'
     masthead_url          = 'http://www.newscientist.com/img/misc/ns_logo.jpg'
     encoding              = 'utf-8'
-    extra_css             = ' body{font-family: Arial,sans-serif} img{margin-bottom: 0.8em} '
+    needs_subscription    = 'optional'
+    extra_css             = """
+                                 body{font-family: Arial,sans-serif}
+                                 img{margin-bottom: 0.8em}
+                                 .quotebx{font-size: x-large; font-weight: bold; margin-right: 2em; margin-left: 2em}
+                            """
 
     conversion_options = {
                           'comment'          : description
@@ -33,15 +38,27 @@ class NewScientist(BasicNewsRecipe):
 
     keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol','blgmaincol','nsblgposts','hldgalcols']})]
 
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.open('http://www.newscientist.com/')
+        if self.username is not None and self.password is not None:
+            br.open('https://www.newscientist.com/user/login?redirectURL=')
+            br.select_form(nr=2)
+            br['loginId' ] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
     remove_tags = [
                      dict(name='div'  , attrs={'class':['hldBd','adline','pnl','infotext' ]})
                     ,dict(name='div'  , attrs={'id'   :['compnl','artIssueInfo','artTools','comments','blgsocial','sharebtns']})
                     ,dict(name='p'    , attrs={'class':['marker','infotext'               ]})
                     ,dict(name='meta' , attrs={'name' :'description'                       })
-                    ,dict(name='a'    , attrs={'rel'  :'tag'                                })
+                    ,dict(name='a'    , attrs={'rel'  :'tag'                               })
+                    ,dict(name=['link','base','meta','iframe','object','embed'])
                   ]
     remove_tags_after = dict(attrs={'class':['nbpcopy','comments']})
-    remove_attributes = ['height','width']
+    remove_attributes = ['height','width','lang']
 
     feeds          = [
                         (u'Latest Headlines'        , u'http://feeds.newscientist.com/science-news'              )
@@ -62,6 +79,8 @@ class NewScientist(BasicNewsRecipe):
         return url + '?full=true&print=true'
 
     def preprocess_html(self, soup):
+        for item in soup.findAll(['quote','quotetext']):
+            item.name='p'
         for tg in soup.findAll('a'):
             if tg.string == 'Home':
                 tg.parent.extract()

resources/recipes/theecocolapse.recipe

@@ -0,0 +1,46 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+theeconomiccollapseblog.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+class TheEconomicCollapse(BasicNewsRecipe):
+    title                 = 'The Economic Collapse'
+    __author__            = 'Darko Miletic'
+    description           = 'Are You Prepared For The Coming Economic Collapse And The Next Great Depression?'
+    publisher             = 'The Economic Collapse'
+    category              = 'news, politics, USA, economy'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    extra_css             = """
+                                body{font-family: Tahoma,Arial,sans-serif }
+                                img{margin-bottom: 0.4em}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [
+                     dict(attrs={'class':'sociable'})
+                    ,dict(name=['iframe','object','embed','meta','link','base'])
+                  ]
+    remove_attributes=['lang','onclick','width','height']
+    keep_only_tags=[dict(attrs={'class':['post-headline','post-bodycopy clearfix','']})]
+
+    feeds = [(u'Posts', u'http://theeconomiccollapseblog.com/feed')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)
+

resources/recipes/theeconomictimes_india.recipe

@@ -19,20 +19,22 @@ class TheEconomicTimes(BasicNewsRecipe):
     simultaneous_downloads = 1
     encoding               = 'utf-8'
     language               = 'en_IN'
-    publication_type       = 'newspaper'    
+    publication_type       = 'newspaper'
     masthead_url           = 'http://economictimes.indiatimes.com/photo/2676871.cms'
-    extra_css              = """ body{font-family: Arial,Helvetica,sans-serif} 
+    extra_css              = """
-                                .heading1{font-size: xx-large; font-weight: bold} """
+                                 body{font-family: Arial,Helvetica,sans-serif}
-    
+                             """
+
     conversion_options = {
                           'comment'          : description
                         , 'tags'             : category
                         , 'publisher'        : publisher
                         , 'language'         : language
                         }
-                        
+
-    keep_only_tags = [dict(attrs={'class':['heading1','headingnext','Normal']})]
+    keep_only_tags = [dict(attrs={'class':'printdiv'})]
     remove_tags    = [dict(name=['object','link','embed','iframe','base','table','meta'])]
+    remove_attributes = ['name']
 
     feeds          = [(u'All articles', u'http://economictimes.indiatimes.com/rssfeedsdefault.cms')]
 
@@ -48,5 +50,5 @@ class TheEconomicTimes(BasicNewsRecipe):
 
     def preprocess_html(self, soup):
         for item in soup.findAll(style=True):
-            del item['style']    
+            del item['style']
         return self.adeify_images(soup)

src/calibre/customize/conversion.py

@@ -294,3 +294,8 @@ class OutputFormatPlugin(Plugin):
         '''
         raise NotImplementedError
 
+    @property
+    def is_periodical(self):
+        return self.oeb.metadata.publication_type and \
+            unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:')
+

src/calibre/customize/profiles.py

@@ -583,7 +583,8 @@ class KindleDXOutput(OutputProfile):
     # Screen size is a best guess
     screen_size               = (744, 1022)
     dpi                       = 150.0
-    comic_screen_size         = (741, 1022)
+    comic_screen_size = (771, 1116)
+    #comic_screen_size         = (741, 1022)
     supports_mobi_indexing = True
     periodical_date_in_title = False
     mobi_ems_per_blockquote = 2.0

src/calibre/devices/cybook/driver.py

@@ -42,7 +42,7 @@ class CYBOOK(USBMS):
     DELETE_EXTS = ['.mbp', '.dat', '.bin', '_6090.t2b', '.thn']
     SUPPORTS_SUB_DIRS = True
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
             coverdata = coverdata[2]

src/calibre/devices/hanvon/driver.py

@@ -77,7 +77,7 @@ class ALEX(N516):
         name = os.path.splitext(os.path.basename(file_abspath))[0] + '.png'
         return os.path.join(base, 'covers', name)
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         from calibre.ebooks import calibre_cover
         from calibre.utils.magick.draw import thumbnail
         coverdata = getattr(metadata, 'thumbnail', None)
@@ -129,7 +129,7 @@ class AZBOOKA(ALEX):
     def can_handle(self, device_info, debug=False):
         return not is_alex(device_info)
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         pass
 
 class EB511(USBMS):

src/calibre/devices/misc.py

@@ -102,7 +102,7 @@ class PDNOVEL(USBMS):
     DELETE_EXTS = ['.jpg', '.jpeg', '.png']
 
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
             with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:

src/calibre/devices/nook/driver.py

@@ -45,7 +45,7 @@ class NOOK(USBMS):
     DELETE_EXTS = ['.jpg']
     SUPPORTS_SUB_DIRS = True
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         try:
             from PIL import Image, ImageDraw
             Image, ImageDraw

src/calibre/devices/prs505/__init__.py

@@ -2,5 +2,11 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 MEDIA_XML = 'database/cache/media.xml'
+MEDIA_EXT = 'database/cache/cacheExt.xml'
 
 CACHE_XML = 'Sony Reader/database/cache.xml'
+CACHE_EXT = 'Sony Reader/database/cacheExt.xml'
+
+MEDIA_THUMBNAIL = 'database/thumbnail'
+CACHE_THUMBNAIL = 'Sony Reader/database/thumbnail'
+

src/calibre/devices/prs505/driver.py

@@ -9,10 +9,10 @@ Device driver for the SONY devices
 import os, time, re
 
 from calibre.devices.usbms.driver import USBMS, debug_print
-from calibre.devices.prs505 import MEDIA_XML
+from calibre.devices.prs505 import MEDIA_XML, MEDIA_EXT, CACHE_XML, CACHE_EXT, \
-from calibre.devices.prs505 import CACHE_XML
+            MEDIA_THUMBNAIL, CACHE_THUMBNAIL
 from calibre.devices.prs505.sony_cache import XMLCache
-from calibre import __appname__
+from calibre import __appname__, prints
 from calibre.devices.usbms.books import CollectionsBookList
 
 class PRS505(USBMS):
@@ -66,6 +66,8 @@ class PRS505(USBMS):
     plugboard = None
     plugboard_func = None
 
+    THUMBNAIL_HEIGHT = 200
+
     def windows_filter_pnp_id(self, pnp_id):
         return '_LAUNCHER' in pnp_id
 
@@ -116,20 +118,21 @@ class PRS505(USBMS):
         return fname
 
     def initialize_XML_cache(self):
-        paths, prefixes = {}, {}
+        paths, prefixes, ext_paths = {}, {}, {}
-        for prefix, path, source_id in [
+        for prefix, path, ext_path, source_id in [
-                ('main', MEDIA_XML, 0),
+                ('main', MEDIA_XML, MEDIA_EXT, 0),
-                ('card_a', CACHE_XML, 1),
+                ('card_a', CACHE_XML, CACHE_EXT, 1),
-                ('card_b', CACHE_XML, 2)
+                ('card_b', CACHE_XML, CACHE_EXT, 2)
                 ]:
             prefix = getattr(self, '_%s_prefix'%prefix)
             if prefix is not None and os.path.exists(prefix):
                 paths[source_id] = os.path.join(prefix, *(path.split('/')))
+                ext_paths[source_id] = os.path.join(prefix, *(ext_path.split('/')))
                 prefixes[source_id] = prefix
                 d = os.path.dirname(paths[source_id])
                 if not os.path.exists(d):
                     os.makedirs(d)
-        return XMLCache(paths, prefixes, self.settings().use_author_sort)
+        return XMLCache(paths, ext_paths, prefixes, self.settings().use_author_sort)
 
     def books(self, oncard=None, end_session=True):
         debug_print('PRS505: starting fetching books for card', oncard)
@@ -174,3 +177,31 @@ class PRS505(USBMS):
     def set_plugboards(self, plugboards, pb_func):
         self.plugboards = plugboards
         self.plugboard_func = pb_func
+
+    def upload_cover(self, path, filename, metadata, filepath):
+        if metadata.thumbnail and metadata.thumbnail[-1]:
+            path = path.replace('/', os.sep)
+            is_main = path.startswith(self._main_prefix)
+            thumbnail_dir = MEDIA_THUMBNAIL if is_main else CACHE_THUMBNAIL
+            prefix = None
+            if is_main:
+                prefix = self._main_prefix
+            else:
+                if self._card_a_prefix and \
+                    path.startswith(self._card_a_prefix):
+                    prefix = self._card_a_prefix
+                elif self._card_b_prefix and \
+                        path.startswith(self._card_b_prefix):
+                    prefix = self._card_b_prefix
+            if prefix is None:
+                prints('WARNING: Failed to find prefix for:', filepath)
+                return
+            thumbnail_dir = os.path.join(prefix, *thumbnail_dir.split('/'))
+
+            relpath = os.path.relpath(filepath, prefix)
+            thumbnail_dir = os.path.join(thumbnail_dir, relpath)
+            if not os.path.exists(thumbnail_dir):
+                os.makedirs(thumbnail_dir)
+            with open(os.path.join(thumbnail_dir, 'main_thumbnail.jpg'), 'wb') as f:
+                f.write(metadata.thumbnail[-1])
+

src/calibre/devices/prs505/sony_cache.py

@@ -9,6 +9,7 @@ import os, time
 from base64 import b64decode
 from uuid import uuid4
 from lxml import etree
+from datetime import date
 
 from calibre import prints, guess_type, isbytestring
 from calibre.devices.errors import DeviceError
@@ -18,6 +19,20 @@ from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata import authors_to_string, title_sort, \
                                     authors_to_sort_string
 
+'''
+cahceExt.xml
+
+Periodical identifier sample from a PRS-650:
+
+<?xml version="1.0" encoding="UTF-8"?>
+<cacheExt xmlns="http://www.sony.com/xmlns/product/prs/device/1">
+    <text conformsTo="http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0" periodicalName="The Atlantic" description="Current affairs and politics focussed on the US" publicationDate="Tue, 19 Oct 2010 00:00:00 GMT" path="database/media/books/calibre/Atlantic [Mon, 18 Oct 2010], The - calibre_1701.epub">
+        <thumbnail width="167" height="217">main_thumbnail.jpg</thumbnail>
+    </text>
+</cacheExt>
+
+'''
+
 # Utility functions {{{
 EMPTY_CARD_CACHE = '''\
 <?xml version="1.0" encoding="UTF-8"?>
@@ -25,6 +40,12 @@ EMPTY_CARD_CACHE = '''\
 </cache>
 '''
 
+EMPTY_EXT_CACHE = '''\
+<?xml version="1.0" encoding="UTF-8"?>
+<cacheExt xmlns="http://www.sony.com/xmlns/product/prs/device/1">
+</cacheExt>
+'''
+
 MIME_MAP   = {
                 "lrf" : "application/x-sony-bbeb",
                 'lrx' : 'application/x-sony-bbeb',
@@ -63,7 +84,7 @@ def uuid():
 
 class XMLCache(object):
 
-    def __init__(self, paths, prefixes, use_author_sort):
+    def __init__(self, paths, ext_paths, prefixes, use_author_sort):
         if DEBUG:
             debug_print('Building XMLCache...', paths)
         self.paths = paths
@@ -76,8 +97,8 @@ class XMLCache(object):
         for source_id, path in paths.items():
             if source_id == 0:
                 if not os.path.exists(path):
-                    raise DeviceError('The SONY XML cache media.xml does not exist. Try'
+                    raise DeviceError(('The SONY XML cache %r does not exist. Try'
-                        ' disconnecting and reconnecting your reader.')
+                        ' disconnecting and reconnecting your reader.')%repr(path))
                 with open(path, 'rb') as f:
                     raw = f.read()
             else:
@@ -85,14 +106,34 @@ class XMLCache(object):
                 if os.access(path, os.R_OK):
                     with open(path, 'rb') as f:
                         raw = f.read()
+
             self.roots[source_id] = etree.fromstring(xml_to_unicode(
                         raw, strip_encoding_pats=True, assume_utf8=True,
                         verbose=DEBUG)[0],
                         parser=parser)
             if self.roots[source_id] is None:
-                raise Exception(('The SONY database at %s is corrupted. Try '
+                raise Exception(('The SONY database at %r is corrupted. Try '
                         ' disconnecting and reconnecting your reader.')%path)
 
+        self.ext_paths, self.ext_roots = {}, {}
+        for source_id, path in ext_paths.items():
+            if not os.path.exists(path):
+                try:
+                    with open(path, 'wb') as f:
+                        f.write(EMPTY_EXT_CACHE)
+                except:
+                    pass
+            if os.access(path, os.W_OK):
+                try:
+                    with open(path, 'rb') as f:
+                        self.ext_roots[source_id] = etree.fromstring(
+                                xml_to_unicode(f.read(),
+                                    strip_encoding_pats=True, assume_utf8=True,
+                                    verbose=DEBUG)[0], parser=parser)
+                        self.ext_paths[source_id] = path
+                except:
+                    pass
+
         # }}}
 
         recs = self.roots[0].xpath('//*[local-name()="records"]')
@@ -352,12 +393,18 @@ class XMLCache(object):
             debug_print('Updating XML Cache:', i)
             root = self.record_roots[i]
             lpath_map = self.build_lpath_map(root)
+            ext_root = self.ext_roots[i] if i in self.ext_roots else None
+            ext_lpath_map = None
+            if ext_root is not None:
+                ext_lpath_map = self.build_lpath_map(ext_root)
             gtz_count = ltz_count = 0
             use_tz_var = False
             for book in booklist:
                 path = os.path.join(self.prefixes[i], *(book.lpath.split('/')))
                 record = lpath_map.get(book.lpath, None)
+                created = False
                 if record is None:
+                    created = True
                     record = self.create_text_record(root, i, book.lpath)
                 if plugboard is not None:
                     newmi = book.deepcopy_metadata()
@@ -373,6 +420,13 @@ class XMLCache(object):
                 if book.device_collections is None:
                     book.device_collections = []
                 book.device_collections = playlist_map.get(book.lpath, [])
+
+                if created and ext_root is not None and \
+                        ext_lpath_map.get(book.lpath, None) is None:
+                    ext_record = self.create_ext_text_record(ext_root, i,
+                            book.lpath, book.thumbnail)
+                    self.periodicalize_book(book, ext_record)
+
             debug_print('Timezone votes: %d GMT, %d LTZ, use_tz_var=%s'%
                                         (gtz_count, ltz_count, use_tz_var))
             self.update_playlists(i, root, booklist, collections_attributes)
@@ -386,6 +440,47 @@ class XMLCache(object):
         self.fix_ids()
         debug_print('Finished update')
 
+    def is_sony_periodical(self, book):
+        if _('News') not in book.tags:
+            return False
+        if not book.lpath.lower().endswith('.epub'):
+            return False
+        if book.pubdate.date() < date(2010, 10, 17):
+            return False
+        return True
+
+    def periodicalize_book(self, book, record):
+        if not self.is_sony_periodical(book):
+            return
+        record.set('conformsTo',
+            "http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0")
+
+        record.set('description', '')
+
+        name = None
+        if '[' in book.title:
+            name = book.title.split('[')[0].strip()
+            if len(name) < 4:
+                name = None
+        if not name:
+            try:
+                name = [t for t in book.tags if t != _('News')][0]
+            except:
+                name = None
+
+        if not name:
+            name = book.title
+
+        record.set('periodicalName', name)
+
+        try:
+            pubdate = strftime(book.pubdate.utctimetuple(),
+                    zone=lambda x : x)
+            record.set('publicationDate', pubdate)
+        except:
+            pass
+
+
     def rebuild_collections(self, booklist, bl_index):
         if bl_index not in self.record_roots:
             return
@@ -472,6 +567,25 @@ class XMLCache(object):
         root.append(ans)
         return ans
 
+    def create_ext_text_record(self, root, bl_id, lpath, thumbnail):
+        namespace = root.nsmap[None]
+        attrib = { 'path': lpath }
+        ans = root.makeelement('{%s}text'%namespace, attrib=attrib,
+                nsmap=root.nsmap)
+        ans.tail = '\n'
+        root[-1].tail = '\n' + '\t'
+        root.append(ans)
+        if thumbnail and thumbnail[-1]:
+            ans.text = '\n' + '\t\t'
+            t = root.makeelement('{%s}thumbnail'%namespace,
+                attrib={'width':str(thumbnail[0]), 'height':str(thumbnail[1])},
+                nsmap=root.nsmap)
+            t.text = 'main_thumbnail.jpg'
+            ans.append(t)
+            t.tail = '\n\t'
+        return ans
+
+
     def update_text_record(self, record, book, path, bl_index,
                            gtz_count, ltz_count, use_tz_var):
         '''
@@ -589,6 +703,18 @@ class XMLCache(object):
                     '<?xml version="1.0" encoding="UTF-8"?>')
             with open(path, 'wb') as f:
                 f.write(raw)
+
+        for i, path in self.ext_paths.items():
+            try:
+                raw = etree.tostring(self.ext_roots[i], encoding='UTF-8',
+                    xml_declaration=True)
+            except:
+                continue
+            raw = raw.replace("<?xml version='1.0' encoding='UTF-8'?>",
+                    '<?xml version="1.0" encoding="UTF-8"?>')
+            with open(path, 'wb') as f:
+                f.write(raw)
+
     # }}}
 
     # Utility methods {{{

src/calibre/devices/udisks.py

@@ -5,8 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import dbus
+import dbus, os
-import os
 
 def node_mountpoint(node):
 
@@ -56,15 +55,6 @@ class UDisks(object):
         parent = device_node_path
         while parent[-1] in '0123456789':
             parent = parent[:-1]
-        devices = [str(x) for x in self.main.EnumerateDeviceFiles()]
-        for d in devices:
-            if d.startswith(parent) and d != parent:
-                try:
-                    self.unmount(d)
-                except:
-                    import traceback
-                    print 'Failed to unmount:', d
-                    traceback.print_exc()
         d = self.device(parent)
         d.DriveEject([])
 
@@ -76,13 +66,19 @@ def eject(node_path):
     u = UDisks()
     u.eject(node_path)
 
+def umount(node_path):
+    u = UDisks()
+    u.unmount(node_path)
+
 if __name__ == '__main__':
     import sys
     dev = sys.argv[1]
     print 'Testing with node', dev
     u = UDisks()
     print 'Mounted at:', u.mount(dev)
-    print 'Ejecting'
+    print 'Unmounting'
+    u.unmount(dev)
+    print 'Ejecting:'
     u.eject(dev)
 
 

src/calibre/devices/usbms/books.py

@@ -99,6 +99,13 @@ class CollectionsBookList(BookList):
     def supports_collections(self):
         return True
 
+    def in_category_sort_rules(self, attr):
+        sorts = tweaks['sony_collection_sorting_rules']
+        for attrs,sortattr in sorts:
+            if attr in attrs or '*' in attrs:
+                return sortattr
+        return None
+
     def compute_category_name(self, attr, category, field_meta):
         renames = tweaks['sony_collection_renaming_rules']
         attr_name = renames.get(attr, None)
@@ -116,6 +123,7 @@ class CollectionsBookList(BookList):
         from calibre.devices.usbms.driver import debug_print
         debug_print('Starting get_collections:', prefs['manage_device_metadata'])
         debug_print('Renaming rules:', tweaks['sony_collection_renaming_rules'])
+        debug_print('Sorting rules:', tweaks['sony_collection_sorting_rules'])
 
         # Complexity: we can use renaming rules only when using automatic
         # management. Otherwise we don't always have the metadata to make the
@@ -171,6 +179,7 @@ class CollectionsBookList(BookList):
                 else:
                     val = [val]
 
+                sort_attr = self.in_category_sort_rules(attr)
                 for category in val:
                     is_series = False
                     if doing_dc:
@@ -199,22 +208,41 @@ class CollectionsBookList(BookList):
 
                     if cat_name not in collections:
                         collections[cat_name] = {}
-                    if is_series:
+                    if use_renaming_rules and sort_attr:
+                        sort_val = book.get(sort_attr, None)
+                        collections[cat_name][lpath] = \
+                                (book, sort_val, book.get('title_sort', 'zzzz'))
+                    elif is_series:
                         if doing_dc:
                             collections[cat_name][lpath] = \
-                                    (book, book.get('series_index', sys.maxint))
+                                (book, book.get('series_index', sys.maxint), '')
                         else:
                             collections[cat_name][lpath] = \
-                                    (book, book.get(attr+'_index', sys.maxint))
+                                (book, book.get(attr+'_index', sys.maxint), '')
                     else:
                         if lpath not in collections[cat_name]:
                             collections[cat_name][lpath] = \
-                                (book, book.get('title_sort', 'zzzz'))
+                                (book, book.get('title_sort', 'zzzz'), '')
         # Sort collections
         result = {}
+
+        def none_cmp(xx, yy):
+            x = xx[1]
+            y = yy[1]
+            if x is None and y is None:
+                return cmp(xx[2], yy[2])
+            if x is None:
+                return 1
+            if y is None:
+                return -1
+            c = cmp(x, y)
+            if c != 0:
+                return c
+            return cmp(xx[2], yy[2])
+
         for category, lpaths in collections.items():
             books = lpaths.values()
-            books.sort(cmp=lambda x,y:cmp(x[1], y[1]))
+            books.sort(cmp=none_cmp)
             result[category] = [x[0] for x in books]
         return result
 

src/calibre/devices/usbms/device.py

@@ -523,7 +523,8 @@ class Device(DeviceConfig, DevicePlugin):
                     devnodes.append(node)
 
         devnodes += list(repeat(None, 3))
-        ans = tuple(['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]])
+        ans = ['/dev/'+x if ok.get(x, False) else None for x in devnodes[:3]]
+        ans.sort(key=lambda x: x[5:] if x else 'zzzzz')
         return self.linux_swap_drives(ans)
 
     def linux_swap_drives(self, drives):
@@ -732,24 +733,36 @@ class Device(DeviceConfig, DevicePlugin):
                     pass
 
     def eject_linux(self):
-        try:
+        from calibre.devices.udisks import eject, umount
-            from calibre.devices.udisks import eject
+        drives = [d for d in self.find_device_nodes() if d]
-            return eject(self._linux_main_device_node)
+        for d in drives:
-        except:
+            try:
-            pass
+                umount(d)
-        drives = self.find_device_nodes()
+            except:
+                pass
+        failures = False
+        for d in drives:
+            try:
+                eject(d)
+            except Exception, e:
+                print 'Udisks eject call for:', d, 'failed:'
+                print '\t', e
+                failures = True
+
+        if not failures:
+            return
+
         for drive in drives:
-            if drive:
+            cmd = 'calibre-mount-helper'
-                cmd = 'calibre-mount-helper'
+            if getattr(sys, 'frozen_path', False):
-                if getattr(sys, 'frozen_path', False):
+                cmd = os.path.join(sys.frozen_path, cmd)
-                    cmd = os.path.join(sys.frozen_path, cmd)
+            cmd = [cmd, 'eject']
-                cmd = [cmd, 'eject']
+            mp = getattr(self, "_linux_mount_map", {}).get(drive,
-                mp = getattr(self, "_linux_mount_map", {}).get(drive,
+                    'dummy/')[:-1]
-                        'dummy/')[:-1]
+            try:
-                try:
+                subprocess.Popen(cmd + [drive, mp]).wait()
-                    subprocess.Popen(cmd + [drive, mp]).wait()
+            except:
-                except:
+                pass
-                    pass
 
     def eject(self):
         if islinux:

src/calibre/devices/usbms/driver.py

@@ -186,7 +186,8 @@ class USBMS(CLI, Device):
             self.put_file(infile, filepath, replace_file=True)
             try:
                 self.upload_cover(os.path.dirname(filepath),
-                                  os.path.splitext(os.path.basename(filepath))[0], mdata)
+                                  os.path.splitext(os.path.basename(filepath))[0],
+                                  mdata, filepath)
             except: # Failure to upload cover is not catastrophic
                 import traceback
                 traceback.print_exc()
@@ -197,14 +198,15 @@ class USBMS(CLI, Device):
         debug_print('USBMS: finished uploading %d books'%(len(files)))
         return zip(paths, cycle([on_card]))
 
-    def upload_cover(self, path, filename, metadata):
+    def upload_cover(self, path, filename, metadata, filepath):
         '''
         Upload book cover to the device. Default implementation does nothing.
 
-        :param path: the full path were the associated book is located.
+        :param path: The full path to the directory where the associated book is located.
-        :param filename: the name of the book file without the extension.
+        :param filename: The name of the book file without the extension.
         :param metadata: metadata belonging to the book. Use metadata.thumbnail
                          for cover
+        :param filepath: The full path to the ebook file
 
         '''
         pass

src/calibre/ebooks/epub/__init__.py

@@ -15,22 +15,30 @@ def rules(stylesheets):
                 if r.type == r.STYLE_RULE:
                     yield r
 
-def initialize_container(path_to_container, opf_name='metadata.opf'):
+def initialize_container(path_to_container, opf_name='metadata.opf',
+        extra_entries=[]):
     '''
     Create an empty EPUB document, with a default skeleton.
     '''
-    CONTAINER='''\
+    rootfiles = ''
+    for path, mimetype, _ in extra_entries:
+        rootfiles += u'<rootfile full-path="{0}" media-type="{1}"/>'.format(
+                path, mimetype)
+    CONTAINER = u'''\
 <?xml version="1.0"?>
 <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
    <rootfiles>
-      <rootfile full-path="%s" media-type="application/oebps-package+xml"/>
+      <rootfile full-path="{0}" media-type="application/oebps-package+xml"/>
+      {extra_entries}
    </rootfiles>
 </container>
-    '''%opf_name
+    '''.format(opf_name, extra_entries=rootfiles).encode('utf-8')
     zf = ZipFile(path_to_container, 'w')
     zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_STORED)
     zf.writestr('META-INF/', '', 0700)
     zf.writestr('META-INF/container.xml', CONTAINER)
+    for path, _, data in extra_entries:
+        zf.writestr(path, data)
     return zf
 
 

src/calibre/ebooks/epub/input.py

@@ -108,6 +108,27 @@ class EPUBInput(InputFormatPlugin):
             open('calibre_raster_cover.jpg', 'wb').write(
                 renderer)
 
+    def find_opf(self):
+        def attr(n, attr):
+            for k, v in n.attrib.items():
+                if k.endswith(attr):
+                    return v
+        try:
+            with open('META-INF/container.xml') as f:
+                root = etree.fromstring(f.read())
+                for r in root.xpath('//*[local-name()="rootfile"]'):
+                    if attr(r, 'media-type') != "application/oebps-package+xml":
+                        continue
+                    path = attr(r, 'full-path')
+                    if not path:
+                        continue
+                    path = os.path.join(os.getcwdu(), *path.split('/'))
+                    if os.path.exists(path):
+                        return path
+        except:
+            import traceback
+            traceback.print_exc()
+
     def convert(self, stream, options, file_ext, log, accelerators):
         from calibre.utils.zipfile import ZipFile
         from calibre import walk
@@ -116,12 +137,13 @@ class EPUBInput(InputFormatPlugin):
         zf = ZipFile(stream)
         zf.extractall(os.getcwd())
         encfile = os.path.abspath(os.path.join('META-INF', 'encryption.xml'))
-        opf = None
+        opf = self.find_opf()
-        for f in walk(u'.'):
+        if opf is None:
-            if f.lower().endswith('.opf') and '__MACOSX' not in f and \
+            for f in walk(u'.'):
-                    not os.path.basename(f).startswith('.'):
+                if f.lower().endswith('.opf') and '__MACOSX' not in f and \
-                opf = os.path.abspath(f)
+                        not os.path.basename(f).startswith('.'):
-                break
+                    opf = os.path.abspath(f)
+                    break
         path = getattr(stream, 'name', 'stream')
 
         if opf is None:

src/calibre/ebooks/epub/output.py

@@ -106,6 +106,7 @@ class EPUBOutput(OutputFormatPlugin):
     recommendations = set([('pretty_print', True, OptionRecommendation.HIGH)])
 
 
+
     def workaround_webkit_quirks(self): # {{{
         from calibre.ebooks.oeb.base import XPath
         for x in self.oeb.spine:
@@ -183,6 +184,12 @@ class EPUBOutput(OutputFormatPlugin):
 
         with TemporaryDirectory('_epub_output') as tdir:
             from calibre.customize.ui import plugin_for_output_format
+            metadata_xml = None
+            extra_entries = []
+            if self.is_periodical:
+                from calibre.ebooks.epub.periodical import sony_metadata
+                metadata_xml, atom_xml = sony_metadata(oeb)
+                extra_entries = [('atom.xml', 'application/atom+xml', atom_xml)]
             oeb_output = plugin_for_output_format('oeb')
             oeb_output.convert(oeb, tdir, input_plugin, opts, log)
             opf = [x for x in os.listdir(tdir) if x.endswith('.opf')][0]
@@ -194,10 +201,14 @@ class EPUBOutput(OutputFormatPlugin):
                 encryption = self.encrypt_fonts(encrypted_fonts, tdir, uuid)
 
             from calibre.ebooks.epub import initialize_container
-            epub = initialize_container(output_path, os.path.basename(opf))
+            epub = initialize_container(output_path, os.path.basename(opf),
+                    extra_entries=extra_entries)
             epub.add_dir(tdir)
             if encryption is not None:
                 epub.writestr('META-INF/encryption.xml', encryption)
+            if metadata_xml is not None:
+                epub.writestr('META-INF/metadata.xml',
+                        metadata_xml.encode('utf-8'))
             if opts.extract_to is not None:
                 if os.path.exists(opts.extract_to):
                     shutil.rmtree(opts.extract_to)

src/calibre/ebooks/epub/periodical.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from uuid import uuid4
+
+from calibre.constants import __appname__, __version__
+from calibre import strftime, prepare_string_for_xml as xml
+
+SONY_METADATA = u'''\
+<?xml version="1.0" encoding="utf-8"?>
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+		 xmlns:dcterms="http://purl.org/dc/terms/"
+		 xmlns:dc="http://purl.org/dc/elements/1.1/"
+		 xmlns:prs="http://xmlns.sony.net/e-book/prs/">
+	<rdf:Description rdf:about="">
+		<dc:title>{title}</dc:title>
+		<dc:publisher>{publisher}</dc:publisher>
+		<dcterms:alternative>{short_title}</dcterms:alternative>
+		<dcterms:issued>{issue_date}</dcterms:issued>
+		<dc:language>{language}</dc:language>
+		<dcterms:conformsTo rdf:resource="http://xmlns.sony.net/e-book/prs/periodicals/1.0/newspaper/1.0"/>
+		<dcterms:type rdf:resource="http://xmlns.sony.net/e-book/prs/datatype/newspaper"/>
+		<dcterms:type rdf:resource="http://xmlns.sony.net/e-book/prs/datatype/periodical"/>
+	</rdf:Description>
+</rdf:RDF>
+'''
+
+SONY_ATOM = u'''\
+<?xml version="1.0" encoding="utf-8" ?>
+<feed xmlns="http://www.w3.org/2005/Atom"
+    xmlns:dc="http://purl.org/dc/elements/1.1/"
+    xmlns:dcterms="http://purl.org/dc/terms/"
+    xmlns:prs="http://xmlns.sony.net/e-book/prs/"
+    xmlns:media="http://video.search.yahoo.com/mrss"
+    xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+
+<title>{short_title}</title>
+<updated>{updated}</updated>
+<id>{id}</id>
+{entries}
+</feed>
+'''
+
+SONY_ATOM_SECTION = u'''\
+<entry rdf:ID="{title}">
+  <title>{title}</title>
+  <link href="{href}"/>
+  <id>{id}</id>
+  <updated>{updated}</updated>
+  <summary>{desc}</summary>
+  <category term="{short_title}/{title}"
+      scheme="http://xmlns.sony.net/e-book/terms/" label="{title}"/>
+  <dc:type xsi:type="prs:datatype">newspaper/section</dc:type>
+  <dcterms:isReferencedBy rdf:resource=""/>
+</entry>
+'''
+
+SONY_ATOM_ENTRY = u'''\
+<entry>
+  <title>{title}</title>
+  <author><name>{author}</name></author>
+  <link href="{href}"/>
+  <id>{id}</id>
+  <updated>{updated}</updated>
+  <summary>{desc}</summary>
+  <category term="{short_title}/{section_title}"
+      scheme="http://xmlns.sony.net/e-book/terms/" label="{section_title}"/>
+  <dcterms:extent xsi:type="prs:word-count">{word_count}</dcterms:extent>
+  <dc:type xsi:type="prs:datatype">newspaper/article</dc:type>
+  <dcterms:isReferencedBy rdf:resource="#{section_title}"/>
+</entry>
+'''
+
+def sony_metadata(oeb):
+    m = oeb.metadata
+    title = short_title = unicode(m.title[0])
+    publisher = __appname__ + ' ' + __version__
+    try:
+        pt = unicode(oeb.metadata.publication_type[0])
+        short_title = u':'.join(pt.split(':')[2:])
+    except:
+        pass
+
+    try:
+        date = unicode(m.date[0]).split('T')[0]
+    except:
+        date = strftime('%Y-%m-%d')
+    try:
+        language = unicode(m.language[0]).replace('_', '-')
+    except:
+        language = 'en'
+    short_title = xml(short_title, True)
+
+    metadata = SONY_METADATA.format(title=xml(title),
+            short_title=short_title,
+            publisher=xml(publisher), issue_date=xml(date),
+            language=xml(language))
+
+    updated = strftime('%Y-%m-%dT%H:%M:%SZ')
+
+    def cal_id(x):
+        for k, v in x.attrib.items():
+            if k.endswith('scheme') and v == 'uuid':
+                return True
+
+    try:
+        base_id = unicode(list(filter(cal_id, m.identifier))[0])
+    except:
+        base_id = str(uuid4())
+
+    entries = []
+    seen_titles = set([])
+    for i, section in enumerate(oeb.toc):
+        if not section.href:
+            continue
+        secid = 'section%d'%i
+        sectitle = section.title
+        if not sectitle:
+            sectitle = _('Unknown')
+        d = 1
+        bsectitle = sectitle
+        while sectitle in seen_titles:
+            sectitle = bsectitle + ' ' + str(d)
+            d += 1
+        seen_titles.add(sectitle)
+        sectitle = xml(sectitle, True)
+        secdesc = section.description
+        if not secdesc:
+            secdesc = ''
+        secdesc = xml(secdesc)
+        entries.append(SONY_ATOM_SECTION.format(title=sectitle,
+            href=section.href, id=xml(base_id)+'/'+secid,
+            short_title=short_title, desc=secdesc, updated=updated))
+
+        for j, article in enumerate(section):
+            if not article.href:
+                continue
+            atitle = article.title
+            btitle = atitle
+            d = 1
+            while atitle in seen_titles:
+                atitle = btitle + ' ' + str(d)
+                d += 1
+
+            auth = article.author if article.author else ''
+            desc = section.description
+            if not desc:
+                desc = ''
+            aid = 'article%d'%j
+
+            entries.append(SONY_ATOM_ENTRY.format(
+                title=xml(atitle),
+                author=xml(auth),
+                updated=updated,
+                desc=desc,
+                short_title=short_title,
+                section_title=sectitle,
+                href=article.href,
+                word_count=str(1),
+                id=xml(base_id)+'/'+secid+'/'+aid
+            ))
+
+    atom = SONY_ATOM.format(short_title=short_title,
+            entries='\n\n'.join(entries), updated=updated,
+            id=xml(base_id)).encode('utf-8')
+
+    return metadata, atom
+

src/calibre/ebooks/metadata/book/base.py

@@ -43,7 +43,7 @@ class SafeFormat(TemplateFormatter):
             b = self.book.get_user_metadata(key, False)
             if b and b['datatype'] == 'int' and self.book.get(key, 0) == 0:
                 v = ''
-            elif b and b['datatype'] == 'float' and b.get(key, 0.0) == 0.0:
+            elif b and b['datatype'] == 'float' and self.book.get(key, 0.0) == 0.0:
                 v = ''
             else:
                 ign, v = self.book.format_field(key.lower(), series_with_index=False)
@@ -501,7 +501,7 @@ class Metadata(object):
         if key.startswith('#') and key.endswith('_index'):
             tkey = key[:-6] # strip the _index
             cmeta = self.get_user_metadata(tkey, make_copy=False)
-            if cmeta['datatype'] == 'series':
+            if cmeta and cmeta['datatype'] == 'series':
                 if self.get(tkey):
                     res = self.get_extra(tkey)
                     return (unicode(cmeta['name']+'_index'),

src/calibre/ebooks/metadata/opf2.py

@@ -382,11 +382,13 @@ class Guide(ResourceCollection): # {{{
 
 class MetadataField(object):
 
-    def __init__(self, name, is_dc=True, formatter=None, none_is=None):
+    def __init__(self, name, is_dc=True, formatter=None, none_is=None,
+            renderer=lambda x: unicode(x)):
         self.name      = name
         self.is_dc     = is_dc
         self.formatter = formatter
         self.none_is   = none_is
+        self.renderer  = renderer
 
     def __real_get__(self, obj, type=None):
         ans = obj.get_metadata_element(self.name)
@@ -418,7 +420,7 @@ class MetadataField(object):
             return
         if elem is None:
             elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
-        obj.set_text(elem, unicode(val))
+        obj.set_text(elem, self.renderer(val))
 
 
 def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)):
@@ -489,10 +491,11 @@ class OPF(object): # {{{
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=float, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
-    pubdate         = MetadataField('date', formatter=parse_date)
+    pubdate         = MetadataField('date', formatter=parse_date,
+                                    renderer=isoformat)
     publication_type = MetadataField('publication_type', is_dc=False)
     timestamp       = MetadataField('timestamp', is_dc=False,
-                                    formatter=parse_date)
+                                    formatter=parse_date, renderer=isoformat)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True,
@@ -826,11 +829,10 @@ class OPF(object): # {{{
 
         def fset(self, val):
             matches = self.isbn_path(self.metadata)
-            if val is None:
+            if not val:
-                if matches:
+                for x in matches:
-                    for x in matches:
+                    x.getparent().remove(x)
-                        x.getparent().remove(x)
+                return
-                    return
             if not matches:
                 attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'}
                 matches = [self.create_metadata_element('identifier',
@@ -987,11 +989,14 @@ class OPF(object): # {{{
     def smart_update(self, mi, replace_metadata=False):
         for attr in ('title', 'authors', 'author_sort', 'title_sort',
                      'publisher', 'series', 'series_index', 'rating',
-                     'isbn', 'language', 'tags', 'category', 'comments',
+                     'isbn', 'tags', 'category', 'comments',
                      'pubdate'):
             val = getattr(mi, attr, None)
             if val is not None and val != [] and val != (None, None):
                 setattr(self, attr, val)
+        lang = getattr(mi, 'language', None)
+        if lang and lang != 'und':
+            self.language = lang
         temp = self.to_book_metadata()
         temp.smart_update(mi, replace_metadata=replace_metadata)
         self._user_metadata_ = temp.get_all_user_metadata(True)

src/calibre/ebooks/mobi/output.py

@@ -42,11 +42,10 @@ class MOBIOutput(OutputFormatPlugin):
     ])
 
     def check_for_periodical(self):
-        if self.oeb.metadata.publication_type and \
+        if self.is_periodical:
-            unicode(self.oeb.metadata.publication_type[0]).startswith('periodical:'):
+            self.periodicalize_toc()
-                self.periodicalize_toc()
+            self.check_for_masthead()
-                self.check_for_masthead()
+            self.opts.mobi_periodical = True
-                self.opts.mobi_periodical = True
         else:
             self.opts.mobi_periodical = False
 

src/calibre/gui2/custom_column_widgets.py

@@ -429,7 +429,38 @@ class BulkBase(Base):
             self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
 class BulkBool(BulkBase, Bool):
-    pass
+
+    def get_initial_value(self, book_ids):
+        value = None
+        for book_id in book_ids:
+            val = self.db.get_custom(book_id, num=self.col_id, index_is_id=True)
+            if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
+                val = False
+            if value is not None and value != val:
+                return None
+            value = val
+        return value
+
+    def setup_ui(self, parent):
+        self.widgets = [QLabel('&'+self.col_metadata['name']+':', parent),
+                QComboBox(parent)]
+        w = self.widgets[1]
+        items = [_('Yes'), _('No'), _('Undefined')]
+        icons = [I('ok.png'), I('list_remove.png'), I('blank.png')]
+        for icon, text in zip(icons, items):
+            w.addItem(QIcon(icon), text)
+
+    def setter(self, val):
+        val = {None: 2, False: 1, True: 0}[val]
+        self.widgets[1].setCurrentIndex(val)
+
+    def commit(self, book_ids, notify=False):
+        val = self.gui_val
+        val = self.normalize_ui_val(val)
+        if val != self.initial_val:
+            if tweaks['bool_custom_columns_are_tristate'] == 'no' and val is None:
+                val = False
+            self.db.set_custom_bulk(book_ids, val, num=self.col_id, notify=notify)
 
 class BulkInt(BulkBase, Int):
     pass

src/calibre/gui2/dialogs/check_library.py

@@ -11,7 +11,7 @@ from PyQt4.Qt import QDialog, QVBoxLayout, QHBoxLayout, QTreeWidget, QLabel, \
 
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.library.check_library import CheckLibrary, CHECKS
-from calibre.library.database2 import delete_file
+from calibre.library.database2 import delete_file, delete_tree
 from calibre import prints
 
 class Item(QTreeWidgetItem):
@@ -44,14 +44,10 @@ class CheckLibraryDialog(QDialog):
         self.delete = QPushButton('Delete &marked')
         self.delete.setDefault(False)
         self.delete.clicked.connect(self.delete_marked)
-        self.cancel = QPushButton('&Cancel')
-        self.cancel.setDefault(False)
-        self.cancel.clicked.connect(self.reject)
         self.bbox = QDialogButtonBox(self)
-        self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
         self.bbox.addButton(self.check, QDialogButtonBox.ActionRole)
         self.bbox.addButton(self.delete, QDialogButtonBox.ActionRole)
-        self.bbox.addButton(self.cancel, QDialogButtonBox.RejectRole)
+        self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole)
         self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole)
 
         h = QHBoxLayout()
@@ -146,7 +142,11 @@ class CheckLibraryDialog(QDialog):
         for it in items:
             if it.checkState(1):
                 try:
-                    delete_file(os.path.join(self.db.library_path, unicode(it.text(1))))
+                    p = os.path.join(self.db.library_path ,unicode(it.text(1)))
+                    if os.path.isdir(p):
+                        delete_tree(p)
+                    else:
+                        delete_file(p)
                 except:
                     prints('failed to delete',
                             os.path.join(self.db.library_path,

src/calibre/gui2/dialogs/fetch_metadata.py

@@ -196,7 +196,8 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
             if self.model.rowCount() < 1:
                 info_dialog(self, _('No metadata found'),
                      _('No metadata found, try adjusting the title and author '
-                       'or the ISBN key.')).exec_()
+                       'and/or removing the ISBN.')).exec_()
+                self.reject()
                 return
 
             self.matches.setModel(self.model)

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -16,6 +16,7 @@ from calibre.gui2.custom_column_widgets import populate_metadata_page
 from calibre.gui2 import error_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.utils.config import dynamic
+from calibre.utils.titlecase import titlecase
 
 class MyBlockingBusy(QDialog):
 
@@ -50,6 +51,7 @@ class MyBlockingBusy(QDialog):
         self.start()
 
         self.args = args
+        self.series_start_value = None
         self.db = db
         self.ids = ids
         self.error = None
@@ -115,7 +117,7 @@ class MyBlockingBusy(QDialog):
                     aum = [a.strip().replace('|', ',') for a in aum.split(',')]
                     new_title = authors_to_string(aum)
                     if do_title_case:
-                        new_title = new_title.title()
+                        new_title = titlecase(new_title)
                     self.db.set_title(id, new_title, notify=False)
                     title_set = True
                 if title:
@@ -123,7 +125,7 @@ class MyBlockingBusy(QDialog):
                     self.db.set_authors(id, new_authors, notify=False)
             if do_title_case and not title_set:
                 title = self.db.title(id, index_is_id=True)
-                self.db.set_title(id, title.title(), notify=False)
+                self.db.set_title(id, titlecase(title), notify=False)
             if au:
                 self.db.set_authors(id, string_to_authors(au), notify=False)
         elif self.current_phase == 2:
@@ -147,8 +149,10 @@ class MyBlockingBusy(QDialog):
 
             if do_series:
                 if do_series_restart:
-                    next = series_start_value
+                    if self.series_start_value is None:
-                    series_start_value += 1
+                        self.series_start_value = series_start_value
+                    next = self.series_start_value
+                    self.series_start_value += 1
                 else:
                     next = self.db.get_next_series_num_for(series)
                 self.db.set_series(id, series, notify=False, commit=False)
@@ -179,7 +183,7 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
     s_r_functions = {       ''              : lambda x: x,
                             _('Lower Case') : lambda x: x.lower(),
                             _('Upper Case') : lambda x: x.upper(),
-                            _('Title Case') : lambda x: x.title(),
+                            _('Title Case') : lambda x: titlecase(x),
                     }
 
     s_r_match_modes = [     _('Character match'),

src/calibre/gui2/library/models.py

@@ -783,18 +783,22 @@ class BooksModel(QAbstractTableModel): # {{{
                     self.db.set_rating(id, val)
                 elif column == 'series':
                     val = val.strip()
-                    pat = re.compile(r'\[([.0-9]+)\]')
+                    if not val:
-                    match = pat.search(val)
-                    if match is not None:
-                        self.db.set_series_index(id, float(match.group(1)))
-                        val = pat.sub('', val).strip()
-                    elif val:
-                        if tweaks['series_index_auto_increment'] == 'next':
-                            ni = self.db.get_next_series_num_for(val)
-                            if ni != 1:
-                                self.db.set_series_index(id, ni)
-                    if val:
                         self.db.set_series(id, val)
+                        self.db.set_series_index(id, 1.0)
+                    else:
+                        pat = re.compile(r'\[([.0-9]+)\]')
+                        match = pat.search(val)
+                        if match is not None:
+                            self.db.set_series_index(id, float(match.group(1)))
+                            val = pat.sub('', val).strip()
+                        elif val:
+                            if tweaks['series_index_auto_increment'] == 'next':
+                                ni = self.db.get_next_series_num_for(val)
+                                if ni != 1:
+                                    self.db.set_series_index(id, ni)
+                        if val:
+                            self.db.set_series(id, val)
                 elif column == 'timestamp':
                     if val.isNull() or not val.isValid():
                         return False

src/calibre/library/caches.py

@@ -816,6 +816,10 @@ class SortKeyGenerator(object):
                 if val is None:
                     val = ''
                 val = val.lower()
+
+            elif dt == 'bool':
+                val = {True: 1, False: 2, None: 3}.get(val, 3)
+
             yield val
 
     # }}}

src/calibre/library/database2.py

@@ -748,10 +748,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return False
 
     def find_identical_books(self, mi):
-        fuzzy_title_patterns = [(re.compile(pat), repl) for pat, repl in
+        fuzzy_title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in
                 [
                     (r'[\[\](){}<>\'";,:#]', ''),
-                    (r'^(the|a|an) ', ''),
+                    (tweaks.get('title_sort_articles', r'^(a|the|an)\s+'), ''),
                     (r'[-._]', ' '),
                     (r'\s+', ' ')
                 ]

src/calibre/library/restore.py

@@ -71,9 +71,17 @@ class Restore(Thread):
 
         if self.conflicting_custom_cols:
             ans += '\n\n'
-            ans += 'The following custom columns were not fully restored:\n'
+            ans += 'The following custom columns have conflicting definitions ' \
+                    'and were not fully restored:\n'
             for x in self.conflicting_custom_cols:
                 ans += '\t#'+x+'\n'
+                ans += '\tused:\t%s, %s, %s, %s\n'%(self.custom_columns[x][1],
+                                                    self.custom_columns[x][2],
+                                                    self.custom_columns[x][3],
+                                                    self.custom_columns[x][5])
+                for coldef in self.conflicting_custom_cols[x]:
+                    ans += '\tother:\t%s, %s, %s, %s\n'%(coldef[1], coldef[2],
+                                                         coldef[3], coldef[5])
 
         if self.mismatched_dirs:
             ans += '\n\n'
@@ -152,7 +160,7 @@ class Restore(Thread):
 
     def create_cc_metadata(self):
         self.books.sort(key=itemgetter('timestamp'))
-        m = {}
+        self.custom_columns = {}
         fields = ('label', 'name', 'datatype', 'is_multiple', 'is_editable',
                     'display')
         for b in self.books:
@@ -168,16 +176,17 @@ class Restore(Thread):
                 if len(args) == len(fields):
                     # TODO: Do series type columns need special handling?
                     label = cfm['label']
-                    if label in m and args != m[label]:
+                    if label in self.custom_columns and args != self.custom_columns[label]:
                         if label not in self.conflicting_custom_cols:
-                            self.conflicting_custom_cols[label] = set([m[label]])
+                            self.conflicting_custom_cols[label] = []
-                        self.conflicting_custom_cols[label].add(args)
+                        if self.custom_columns[label] not in self.conflicting_custom_cols[label]:
-                    m[cfm['label']] = args
+                            self.conflicting_custom_cols[label].append(self.custom_columns[label])
+                    self.custom_columns[label] = args
 
         db = RestoreDatabase(self.library_path)
-        self.progress_callback(None, len(m))
+        self.progress_callback(None, len(self.custom_columns))
-        if len(m):
+        if len(self.custom_columns):
-            for i,args in enumerate(m.values()):
+            for i,args in enumerate(self.custom_columns.values()):
                 db.create_custom_column(*args)
                 self.progress_callback(_('creating custom column ')+args[0], i+1)
         db.conn.close()

src/calibre/library/save_to_disk.py

@@ -131,15 +131,14 @@ class SafeFormat(TemplateFormatter):
                     self.vformat(b['display']['composite_template'], [], kwargs)
                 return self.composite_values[key]
             if key in kwargs:
-                return kwargs[key].replace('/', '_').replace('\\', '_')
+                val = kwargs[key]
+                return val.replace('/', '_').replace('\\', '_')
             return ''
         except:
             if DEBUG:
                 traceback.print_exc()
             return key
 
-safe_formatter = SafeFormat()
-
 def get_components(template, mi, id, timefmt='%b %Y', length=250,
         sanitize_func=ascii_filename, replace_whitespace=False,
         to_lowercase=False):
@@ -173,17 +172,22 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
     custom_metadata = mi.get_all_user_metadata(make_copy=False)
     for key in custom_metadata:
         if key in format_args:
+            cm = custom_metadata[key]
             ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't...
-            if custom_metadata[key]['datatype'] == 'series':
+            if cm['datatype'] == 'series':
                 format_args[key] = tsfmt(format_args[key])
                 if key+'_index' in format_args:
                     format_args[key+'_index'] = fmt_sidx(format_args[key+'_index'])
-            elif custom_metadata[key]['datatype'] == 'datetime':
+            elif cm['datatype'] == 'datetime':
                 format_args[key] = strftime(timefmt, format_args[key].timetuple())
-            elif custom_metadata[key]['datatype'] == 'bool':
+            elif cm['datatype'] == 'bool':
                 format_args[key] = _('yes') if format_args[key] else _('no')
-
+            elif cm['datatype'] in ['int', 'float']:
-    components = safe_formatter.safe_format(template, format_args,
+                if format_args[key] != 0:
+                    format_args[key] = unicode(format_args[key])
+                else:
+                    format_args[key] = ''
+    components = SafeFormat().safe_format(template, format_args,
                                             'G_C-EXCEPTION!', mi)
     components = [x.strip() for x in components.split('/') if x.strip()]
     components = [sanitize_func(x) for x in components if x]

src/calibre/library/server/base.py

@@ -148,6 +148,7 @@ class LibraryServer(ContentServer, MobileServer, XMLServer, OPDSServer, Cache,
         cherrypy.engine.graceful()
 
     def set_search_restriction(self, restriction):
+        self.search_restriction_name = restriction
         if restriction:
             self.search_restriction = 'search:"%s"'%restriction
         else:

src/calibre/library/server/browse.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import operator, os, json
 from binascii import hexlify, unhexlify
-from urllib import quote
+from urllib import quote, unquote
 
 import cherrypy
 
@@ -116,7 +116,10 @@ def render_rating(rating, container='span', prefix=None): # {{{
 
 # }}}
 
-def get_category_items(category, items, db, datatype): # {{{
+def get_category_items(category, items, restriction, datatype): # {{{
+
+    if category == 'search':
+        items = [x for x in items if x.name != restriction]
 
     def item(i):
         templ = (u'<div title="{4}" class="category-item">'
@@ -165,6 +168,9 @@ class Endpoint(object): # {{{
                     sort_val = cookie[eself.sort_cookie_name].value
                 kwargs[eself.sort_kwarg] = sort_val
 
+            # Remove AJAX caching disabling jquery workaround arg
+            kwargs.pop('_', None)
+
             ans = func(self, *args, **kwargs)
             cherrypy.response.headers['Content-Type'] = eself.mimetype
             updated = self.db.last_modified()
@@ -299,6 +305,7 @@ class BrowseServer(object):
         category_meta = self.db.field_metadata
         cats = [
                 (_('Newest'), 'newest', 'forward.png'),
+                (_('All books'), 'allbooks', 'book.png'),
                 ]
 
         def getter(x):
@@ -370,7 +377,8 @@ class BrowseServer(object):
 
         if len(items) <= self.opts.max_opds_ungrouped_items:
             script = 'false'
-            items = get_category_items(category, items, self.db, datatype)
+            items = get_category_items(category, items,
+                    self.search_restriction_name, datatype)
         else:
             getter = lambda x: unicode(getattr(x, 'sort', x.name))
             starts = set([])
@@ -440,7 +448,8 @@ class BrowseServer(object):
                 entries.append(x)
 
         sort = self.browse_sort_categories(entries, sort)
-        entries = get_category_items(category, entries, self.db, datatype)
+        entries = get_category_items(category, entries,
+                self.search_restriction_name, datatype)
         return json.dumps(entries, ensure_ascii=False)
 
 
@@ -451,6 +460,8 @@ class BrowseServer(object):
             ans = self.browse_toplevel()
         elif category == 'newest':
             raise cherrypy.InternalRedirect('/browse/matches/newest/dummy')
+        elif category == 'allbooks':
+            raise cherrypy.InternalRedirect('/browse/matches/allbooks/dummy')
         else:
             ans = self.browse_category(category, category_sort)
 
@@ -474,20 +485,26 @@ class BrowseServer(object):
 
     @Endpoint(sort_type='list')
     def browse_matches(self, category=None, cid=None, list_sort=None):
+        if list_sort:
+            list_sort = unquote(list_sort)
         if not cid:
             raise cherrypy.HTTPError(404, 'invalid category id: %r'%cid)
         categories = self.categories_cache()
 
-        if category not in categories and category != 'newest':
+        if category not in categories and \
+                category not in ('newest', 'allbooks'):
             raise cherrypy.HTTPError(404, 'category not found')
         fm = self.db.field_metadata
         try:
             category_name = fm[category]['name']
             dt = fm[category]['datatype']
         except:
-            if category != 'newest':
+            if category not in ('newest', 'allbooks'):
                 raise
-            category_name = _('Newest')
+            category_name = {
+                    'newest' : _('Newest'),
+                    'allbooks' : _('All books'),
+            }[category]
             dt = None
 
         hide_sort = 'true' if dt == 'series' else 'false'
@@ -498,8 +515,10 @@ class BrowseServer(object):
             except:
                 raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
         elif category == 'newest':
-            ids = list(self.db.data.iterallids())
+            ids = self.search_cache('')
             hide_sort = 'true'
+        elif category == 'allbooks':
+            ids = self.search_cache('')
         else:
             q = category
             if q == 'news':

src/calibre/library/server/mobile.py

@@ -112,7 +112,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
                 CLASS('thumbnail'))
 
         data = TD()
-        last = None
         for fmt in book['formats'].split(','):
             a = ascii_filename(book['authors'])
             t = ascii_filename(book['title'])
@@ -124,9 +123,11 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
                 ),
                 CLASS('button'))
             s.tail = u''
-            last = s
             data.append(s)
 
+        div = DIV(CLASS('data-container'))
+        data.append(div)
+
         series = u'[%s - %s]'%(book['series'], book['series_index']) \
                 if book['series'] else ''
         tags = u'Tags=[%s]'%book['tags'] if book['tags'] else ''
@@ -137,13 +138,13 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS):
             if val:
                 ctext += '%s=[%s] '%tuple(val.split(':#:'))
 
-        text = u'\u202f%s %s by %s - %s - %s %s %s' % (book['title'], series,
+        first = SPAN(u'\u202f%s %s by %s' % (book['title'], series,
-                book['authors'], book['size'], book['timestamp'], tags, ctext)
+            book['authors']), CLASS('first-line'))
-
+        div.append(first)
-        if last is None:
+        second = SPAN(u'%s - %s %s %s' % ( book['size'],
-            data.text = text
+            book['timestamp'],
-        else:
+            tags, ctext), CLASS('second-line'))
-            last.tail += text
+        div.append(second)
 
         bookt.append(TR(thumbnail, data))
     # }}}
@@ -229,7 +230,7 @@ class MobileServer(object):
                                              no_tag_count=True)
             book['title'] = record[FM['title']]
             for x in ('timestamp', 'pubdate'):
-                book[x] = strftime('%Y/%m/%d %H:%M:%S', record[FM[x]])
+                book[x] = strftime('%b, %Y', record[FM[x]])
             book['id'] = record[FM['id']]
             books.append(book)
             for key in CKEYS:

src/calibre/utils/formatter.py

@@ -7,6 +7,7 @@ Created on 23 Sep 2010
 import re, string, traceback
 
 from calibre.constants import DEBUG
+from calibre.utils.titlecase import titlecase
 
 class TemplateFormatter(string.Formatter):
     '''
@@ -81,7 +82,7 @@ class TemplateFormatter(string.Formatter):
     functions = {
                     'uppercase'     : (0, lambda s,x: x.upper()),
                     'lowercase'     : (0, lambda s,x: x.lower()),
-                    'titlecase'     : (0, lambda s,x: x.title()),
+                    'titlecase'     : (0, lambda s,x: titlecase(x)),
                     'capitalize'    : (0, lambda s,x: x.capitalize()),
                     'contains'      : (3, _contains),
                     'ifempty'       : (1, _ifempty),

src/calibre/web/feeds/news.py

@@ -1104,7 +1104,7 @@ class BasicNewsRecipe(Recipe):
         mi = MetaInformation(title, [__appname__])
         mi.publisher = __appname__
         mi.author_sort = __appname__
-        mi.publication_type = 'periodical:'+self.publication_type
+        mi.publication_type = 'periodical:'+self.publication_type+':'+self.short_title()
         mi.timestamp = nowf()
         mi.comments = self.description
         if not isinstance(mi.comments, unicode):