Sync to trunk.

Changelog.yaml

@@ -4,6 +4,121 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.6.49
+  date: 2010-04-23
+
+  new features:
+    - title: "Support for the SpringDesign Alex and the Nokia 5800XM"
+      tickets: [5215]
+
+    - title: "Justification control is now more sophisticated. You can choose to have either un-justified text,
+              justified text or leave the justification specified in the input document as is."
+      tickets: [4921]
+
+  bug fixes:
+    - title: "Fix regression that broke database integrity checking in 0.6.48"
+      tickets: [5329]
+
+    - title: "Conversion pipeline: Ignore links in the HTML that have quoted non-ASCII characters, since there is no way to decode them correctly."
+      tickets: [5354]
+
+    - title: "Make title casing more intelligent, based on the guidelines for the New York Times style manual"
+      tickets: [3086]
+
+    - title: "MOBI Input: Handle hexadecimal entities used to specify angle brackets"
+      tickets: [5336]
+
+    - title: "Fix rendering of ratings column in linux when using a 'fancy' style."
+
+    - title: "MOBI Input: Don't fail when the MOBI metadata species a cover that does not exist."
+      tickets: [5333]
+
+    - title: "Fix display of covers in the ebook viewer from MOBI and LIT files"
+      tickets: [5342]
+
+    - title: "MOBI Input: Fix regression that broke detection of covers in MOBI files when converting"
+
+    - title: "Restore blank lines in text only comments when displaying the detailed view for a book"
+
+  new recipes:
+    - title: The West Australian, Kurier, Virtual Shackles
+      author: Darko Miletic
+
+    - title: NPR Music Blogs
+      author: cix3
+
+  improved recipes:
+    - New York Review of Books
+    - USA Today
+    - Guardian
+    - La Republica
+
+- version: 0.6.48
+  date: 2010-04-18
+
+  new features:
+    - title: "Add an output profile for the iPad"
+
+    - title: "An option for ejecting the connected device from the system tray icon"
+
+    - title: "Support for the Samsung SNE 60K and Acer Liquid A1"
+
+    - title: "Programmatic auto-numbering of books in a series"
+      tickets: [2418]
+      description: >
+        "Added an option to the bulk metadata edit dialog to automatically sequentially order the
+        selected books in a series, in the order they were selected."
+
+  bug fixes:
+    - title: "Unmount Bebook Mini on OS X instead of eject"
+      tickets: [5269]
+
+    - title: "Perfomance tweaks to improve startup time with large libraries"
+
+    - title: "Fix changing port of content server only takes effect after a restart of calibre"
+      tickets: [5308]
+
+    - title: "News download: Retry article download if temporary failure in DNS. Makes downloading more robust."
+
+    - title: "Handle DRMed EPUB files correctly when getting cover."
+      tickets: [5304]
+
+    - title: "CHM Metadata: Fix bug in cover detection"
+
+    - title: "Detect zip/rar files that are actually comics even if they contain a mix of PNG/JPG files"
+      tickets: [5300]
+
+    - title: "Fix sending multiple files by email causes them to be mixed up. Also show error message when sending by email fails."
+      tickets: [5069]
+
+    - title: "EPUB Output: When rescaling images from a comic use the same screen size as used by the input plugin"
+
+    - title: "EPUB Output: Fix memory leak when converting documents with large collections of images"
+
+    - title: "Fix regression in 0.6.47 that would cause the main window to expand unctrollably horizontally when using long serach terms"
+
+    - title: "Fix periodicals generated by calibre not using kindle back issue archiving"
+      tickets: [5271]
+
+  new recipes:
+    - title: Canard PC
+      author: zorgluf
+
+    - title: FOX News
+      author: Darko Miletic
+
+    - title: Nation and Standard Media Kenya
+      author: Hans Donner
+
+    - title: El Pais Semanal and Axxon
+      author: Darko MIletic
+
+ 
+  improved recipes:
+    - Pescanik
+    - NSPM
+    - The Atlantic
+
 - version: 0.6.47
   date: 2010-04-09
 

resources/recipes/billorielly.recipe

@@ -0,0 +1,85 @@
+#!/usr/bin/env  python
+
+# ebook-convert.exe c:\billorielly.recipe c:\test -vv
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class BillOReilly(BasicNewsRecipe):
+    cover_url             = 'http://images.billoreilly.com/images/headers/borbanner.jpg'
+    title          		    = u"Bill O'Reilly"
+    __author__ 	          = 'Rob Lammert - rob.lammert[at]gmail.com'
+    description		        = u"Articles from Bill O'Reilly's website and his Fox New's website"
+    oldest_article        = 7.0
+    max_articles_per_feed = 100
+    recursions            = 0
+    encoding              = 'utf8'
+    no_stylesheets        = True
+    remove_javascript     = True
+    #use_embedded_content  = False
+
+
+ #   feeds                 = [
+ #     ('Talking Points Memo', u'http://www.foxnews.com/xmlfeed/column/0,5184,19,00.rss'),
+ #     ('No Spin News', u'http://www.billoreilly.com/blog?rss=true&size=50&useBlurbs=true&categoryID=7')
+ #   ]
+
+    def parse_index(self):
+      feeds = []
+
+      articles_shows = self.bo_parse_shows('http://www.billoreilly.com/show?action=tvShowArchive')
+      articles_columns = self.bo_parse_columns('http://www.billoreilly.com/columns')
+
+      if articles_shows:
+        feeds.append(("O'Reilly Factor", articles_shows))
+
+      if articles_columns:
+        feeds.append(("Newspaper Column", articles_columns))
+
+      return feeds
+
+    def bo_parse_shows(self,url):
+      soup = self.index_to_soup(url)
+      links = soup.find(attrs={'class': 'showLinks'})
+
+      current_articles = []
+      counter = 0
+      for lnk in links.findAllNext(attrs={'class': ['showLinks']}):
+        if counter <= 5:
+          title = self.tag_to_string(lnk)
+          url = lnk.get('href', False)
+
+          if not url or not title:
+            continue
+
+          if url.startswith('/'):
+            url = 'http://www.billoreilly.com'+url+'&dest=/pg/jsp/community/tvshowprint.jsp'
+
+          self.log('\t\tFound article:', title)
+          self.log('\t\t\t', url)
+          current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
+          counter += 1
+      return current_articles
+
+    def bo_parse_columns(self,url):
+      soup = self.index_to_soup(url)
+      links = soup.find(attrs={'id': 'bold'})
+
+      current_articles = []
+      counter = 0
+      for lnk in links.findAllNext(attrs={'id': ['bold']}):
+        test = lnk.get('class', False)
+        if counter <= 5 and test == 'defaultLinks':
+          title = self.tag_to_string(lnk)
+          url = lnk.get('href', False)
+
+          if not url or not title:
+            continue
+
+          if url.startswith('/'):
+            url = 'http://www.billoreilly.com'+url+'&printerFriendly=true"'
+
+          self.log('\t\tFound article:', title)
+          self.log('\t\t\t', url)
+          current_articles.append({'title': title, 'url': url, 'description':'', 'date':''})
+          counter += 1
+      return current_articles

resources/recipes/canardpc.recipe

@@ -0,0 +1,15 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1271446252(BasicNewsRecipe):
+    title          = u'CanardPC'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    language = 'fr'
+    __author__ = 'zorgluf'
+
+    feeds          = [(u'CanardPC', u'http://www.canardpc.com/feed.php')]
+    remove_tags_after  = dict(id='auteur_news')
+    remove_tags_before = dict(id='fil_ariane')
+    no_stylesheets = True
+    remove_tags    = [dict(name='a', attrs={'class':'news_tags'}),
+        dict(name='div', attrs={'id':'fil_ariane'})]

resources/recipes/dani.recipe

@@ -0,0 +1,85 @@
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+bhdani.com
+'''
+
+import re
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BHDani(BasicNewsRecipe):
+    title                  = 'Dani'
+    __author__             = 'Darko Miletic'
+    description            = 'On line izdanje najutjecajnijeg bosanskohercegovackog magazina'
+    publisher              = 'd.o.o. CIVITAS'
+    category               = 'dani, bh, bhdani, magazin, sarajevo, bosna, novine, mediji, listovi, news, magazines, weekly'
+    no_stylesheets         = True
+    oldest_article         = 15
+    encoding               = 'cp1250'
+    needs_subscription     = True
+    remove_empty_feeds     = True
+    PREFIX                 = 'http://bhdani.com'
+    INDEX                  = PREFIX + '/'
+    LOGIN                  = PREFIX + '/users/login.asp'
+    use_embedded_content   = False
+    language               = 'bs'
+    publication_type       = 'magazine'
+    extra_css              = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Arial, sans1, sans-serif} .article_description{font-family: Arial, sans1, sans-serif} .plv18{font-size: xx-large; font-weight: bold; color: #5261A9} .crn10{font-weight: bold} '
+
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    remove_attributes = ['height','width','align']
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open(self.INDEX)
+            br.select_form(name='form')
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    remove_tags       = [dict(name=['link','embed','object','iframe','form'])]
+    remove_tags_before= dict(name='div',attrs={'class':'crn10'})
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        link_item = soup.find('img',attrs={'alt':'Naslovna strana'})
+        if link_item:
+           cover_url = self.PREFIX + link_item['src'].replace('&slika=slika120&','&slika=slika400&')
+        return cover_url
+
+    def print_version(self, url):
+        return url.replace('/default.asp?','/print.asp?')
+
+    def parse_index(self):
+        articles = []
+        soup = self.index_to_soup(self.PREFIX)
+        nrtit = soup.find('font',attrs={'class':'broj'})
+        nrtitle = 'Dani'
+        if nrtit:
+           nrtitle += ' ' + self.tag_to_string(nrtit)
+        for item in soup.findAll('a',attrs={'class':['naslov12','menilink2']}):
+            url   = self.PREFIX + item['href']
+            title = self.tag_to_string(item)
+            description = ''
+            date = strftime(self.timefmt)
+            articles.append({
+                              'title'      :title
+                             ,'date'       :date
+                             ,'url'        :url
+                             ,'description':description
+                            })
+        return [(nrtitle, articles)]
+

resources/recipes/guardian.recipe

@@ -119,5 +119,7 @@ class Guardian(BasicNewsRecipe):
             raise NotImplementedError
 
 
+    def postprocess_html(self,soup,first):
+        return soup.findAll('html')[0]
 
 

resources/recipes/kurier.recipe

@@ -0,0 +1,50 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+kurier.at
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Kurier(BasicNewsRecipe):
+    title                 = 'Kurier'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Austria'
+    publisher             = 'KURIER'
+    category              = 'news, politics, Austria'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'cp1252'
+    use_embedded_content  = False
+    language              = 'de_AT'
+    remove_empty_feeds    = True
+    publication_type      = 'newspaper'
+    extra_css             = ' body{font-family: Verdana,Helvetica,sans-serif } img{margin-bottom: 0.4em} .bild_us{font-size: x-small} '
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [dict(attrs={'class':['contenttabs','drucken','versenden','leserbrief','kommentieren','addthis_button']})]
+    keep_only_tags    = [dict(attrs={'id':'content'})]
+    remove_tags_after = dict(attrs={'id':'author'})
+    remove_attributes = ['width','height']
+
+    feeds = [
+               (u'Nachrichten', u'http://kurier.at/rss/nachrichten_nachrichten_rss.xml'   )
+              ,(u'Techno'     , u'http://kurier.at/rss/techno_techno_rss.xml'   )
+              ,(u'Wirtschaft' , u'http://kurier.at/rss/wirtschaft_wirtschaft_rss.xml'   )
+              ,(u'Kultur'     , u'http://kurier.at/rss/kultur_kultur_rss.xml'   )
+              ,(u'Freizeit'   , u'http://kurier.at/rss/freizeit_freizeit_rss.xml'   )
+              ,(u'Wetter'     , u'http://kurier.at/rss/oewetter_rss.xml'   )
+              ,(u'Verkehr'    , u'http://kurier.at/rss/verkehr_rss.xml'   )
+            ]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        return self.adeify_images(soup)

resources/recipes/la_republica.recipe

@@ -22,21 +22,36 @@ class LaRepublica(BasicNewsRecipe):
     language       = 'it'
     timefmt        = '[%a, %d %b, %Y]'
 
-    oldest_article = 1
+    oldest_article = 5
     max_articles_per_feed = 100
     use_embedded_content  = False
     recursion             = 10
 
     remove_javascript = True
+    def get_article_url(self, article):
+        link = article.get('id', article.get('guid', None))
+        if link is None:
+            return article
+        return link
+
+    keep_only_tags     = [dict(name='div', attrs={'class':'articolo'}),
+                          dict(name='div', attrs={'class':'body-text'}),
+                          dict(name='div', attrs={'class':'page-content'}),
+                          dict(name='div', attrs={'id':'contA'})
+                         ]
 
-    keep_only_tags     = [dict(name='div', attrs={'class':'articolo'})]
 
     remove_tags        = [
                             dict(name=['object','link']),
                             dict(name='span',attrs={'class':'linkindice'}),
-                            dict(name='div',attrs={'class':'bottom-mobile'}),
-                            dict(name='div',attrs={'id':['rssdiv','blocco']})
+                            dict(name='div', attrs={'class':'bottom-mobile'}),
+                            dict(name='div', attrs={'id':['rssdiv','blocco']}),
+                            dict(name='div', attrs={'class':'utility'}),
+                            dict(name='div', attrs={'class':'generalbox'})
                          ]
+    remove_tags_after = [
+                            dict(name='div',attrs={'id':'ugc_linkUpload'})
+                        ]
 
     feeds          = [
                        (u'Repubblica Rilievo', u'http://www.repubblica.it/rss/homepage/rss2.0.xml'),
@@ -48,8 +63,12 @@ class LaRepublica(BasicNewsRecipe):
                        (u'Repubblica Tecnologia', u'http://www.repubblica.it/rss/tecnologia/rss2.0.xml'),
                        (u'Repubblica Scuola e Universita', u'http://www.repubblica.it/rss/scuola_e_universita/rss2.0.xml'),
                        (u'Repubblica Ambiente', u'http://www.repubblica.it/rss/ambiente/rss2.0.xml'),
-		       (u'Repubblica Cultura', u'http://www.repubblica.it/rss/spettacoli_e_cultura/rss2.0.xml'),
-		       (u'Repubblica Persone', u'http://www.repubblica.it/rss/persone/rss2.0.xml'),
-		       (u'Repubblica Sport', u'http://www.repubblica.it/rss/sport/rss2.0.xml'),
-		       (u'Repubblica Calcio', u'http://www.repubblica.it/rss/sport/calcio/rss2.0.xml')
-                     ]
+                       (u'Repubblica Cultura', u'http://www.repubblica.it/rss/spettacoli_e_cultura/rss2.0.xml'),
+                       (u'Repubblica Persone', u'http://www.repubblica.it/rss/persone/rss2.0.xml'),
+                       (u'Repubblica Sport', u'http://www.repubblica.it/rss/sport/rss2.0.xml'),
+                       (u'Repubblica Calcio', u'http://www.repubblica.it/rss/sport/calcio/rss2.0.xml'),
+                       (u'Repubblica Motori', u'http://www.repubblica.it/rss/motori/rss2.0.xml'),
+                       (u'Repubblica Roma', u'http://roma.repubblica.it/rss/rss2.0.xml'),
+                       (u'Repubblica Torino', u'http://torino.repubblica.it/rss/rss2.0.xml')
+                      ]
+

resources/recipes/new_york_review_of_books.recipe

@@ -1,3 +1,4 @@
+
 #!/usr/bin/env  python
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
@@ -6,51 +7,81 @@ __docformat__ = 'restructuredtext en'
 '''
 nybooks.com
 '''
+import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from lxml import html
-from calibre.constants import preferred_encoding
 
 class NewYorkReviewOfBooks(BasicNewsRecipe):
-    
+
     title = u'New York Review of Books'
     description = u'Book reviews'
     language = 'en'
 
-    __author__ = 'Kovid Goyal' 
+    __author__ = 'Kovid Goyal'
+
+    no_stylesheets = True
+    no_javascript = True
     needs_subscription = True
-    remove_tags_before = {'id':'container'}
-    remove_tags = [{'class':['noprint', 'ad', 'footer']}, {'id':'right-content'}]
+
+    keep_only_tags = [dict(id='article-body')]
+    remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
+        'center advertisement']})]
+
+    preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
+        m:'<head></head>')]
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
-        if self.username is not None and self.password is not None:
-            br.open('http://www.nybooks.com/register/')
-            br.select_form(name='login')
-            br['email']   = self.username
-            br['password'] = self.password
-            br.submit()
+        br.open('http://www.nybooks.com/account/signin/')
+        br.select_form(nr = 1)
+        br['username'] = self.username
+        br['password'] = self.password
+        br.submit()
         return br
-    
+
+    def print_version(self, url):
+        return url+'?pagination=false'
+
     def parse_index(self):
-        root = html.fromstring(self.browser.open('http://www.nybooks.com/current-issue').read())
-        date = root.xpath('//h4[@class = "date"]')[0]
-        self.timefmt = ' ['+date.text.encode(preferred_encoding)+']'
+        soup = self.index_to_soup('http://www.nybooks.com/current-issue')
+
+        # Find cover
+        sidebar = soup.find(id='sidebar')
+        if sidebar is not None:
+            a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
+            if a is not None:
+                psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
+                cover = psoup.find('img', src=True)
+                self.cover_url = cover['src']
+                self.log('Found cover at:', self.cover_url)
+
+        # Find date
+        div = soup.find(id='page-title')
+        if div is not None:
+            h5 = div.find('h5')
+            if h5 is not None:
+                text = self.tag_to_string(h5)
+                date = text.partition(u'\u2022')[0].strip()
+                self.timefmt = u' [%s]'%date
+                self.log('Issue date:', date)
+
+        # Find TOC
+        toc = soup.find('ul', attrs={'class':'issue-article-list'})
         articles = []
-        for tag in date.itersiblings():
-            if tag.tag == 'h4': break
-            if tag.tag == 'p':
-                if tag.get('class') == 'indented':
-                    articles[-1]['description'] += html.tostring(tag)
-                else:
-                    href = tag.xpath('descendant::a[@href]')[0].get('href')
-                    article = {
-                               'title': u''.join(tag.xpath('descendant::text()')),
-                               'date' : '',
-                               'url'  : 'http://www.nybooks.com'+href,
-                               'description': '',
-                               }
-                    articles.append(article)
-                    
+        for li in toc.findAll('li'):
+            h3 = li.find('h3')
+            title = self.tag_to_string(h3)
+            author = self.tag_to_string(li.find('h4'))
+            title = title + u' (%s)'%author
+            url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
+            desc = ''
+            for p in li.findAll('p'):
+                desc += self.tag_to_string(p)
+            self.log('Found article:', title)
+            self.log('\t', url)
+            self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'',
+                'description':desc})
+
         return [('Current Issue', articles)]
-       
+

resources/recipes/new_york_review_of_books_no_sub.recipe

@@ -6,10 +6,9 @@ __docformat__ = 'restructuredtext en'
 '''
 nybooks.com
 '''
+import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from lxml import html
-from calibre.constants import preferred_encoding
 
 class NewYorkReviewOfBooks(BasicNewsRecipe):
 
@@ -17,57 +16,61 @@ class NewYorkReviewOfBooks(BasicNewsRecipe):
     description = u'Book reviews'
     language = 'en'
 
-    __author__ = 'Kovid Goyal and Sujata Raman'
+    __author__ = 'Kovid Goyal'
 
     no_stylesheets = True
     no_javascript = True
-    remove_tags_before = {'id':'container'}
-    remove_tags = [{'class':['noprint', 'ad', 'footer']}, {'id':'right-content'},
-                    dict(name='img', attrs={'src':"/images/1x1-clear.gif"}),
 
-                   ]
+    keep_only_tags = [dict(id='article-body')]
+    remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
+        'center advertisement']})]
 
-    extra_css = '''
-                    p{font-family:"Times New Roman",Georgia,serif; font-size: 60%;}
-                    .caption{ font-family:"Times New Roman",Georgia,serif; font-size:40%;}
-                    h2{font-family:"Times New Roman",Georgia,serif; font-size:90%;}
-                    a{ color:#003399; }
-                    .reviewed-title{font-family:"Times New Roman",Georgia,serif;font-size : 50%; font-style:italic;}
-                    .reviewed-author{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
-                    .reviewed-info{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
-                    h5{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
-                    .date{font-family:"Times New Roman",Georgia,serif;font-variant:small-caps;font-size : 50%;}
-                    h4{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
-                '''
+    preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
+        m:'<head></head>')]
 
-    def preprocess_html(self, soup):
-
-         for tag in soup.findAll(name=['span',]):
-             tag.name = 'div'
-         for tag in soup.findAll(name=['blockquote',]):
-             tag.name = 'p'
-
-         return soup
+    def print_version(self, url):
+        return url+'?pagination=false'
 
     def parse_index(self):
-        root = html.fromstring(self.browser.open('http://www.nybooks.com/current-issue').read())
-        date = root.xpath('//h4[@class = "date"]')[0]
-        self.timefmt = ' ['+date.text.encode(preferred_encoding)+']'
+        soup = self.index_to_soup('http://www.nybooks.com/current-issue')
+
+        # Find cover
+        sidebar = soup.find(id='sidebar')
+        if sidebar is not None:
+            a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
+            if a is not None:
+                psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
+                cover = psoup.find('img', src=True)
+                self.cover_url = cover['src']
+                self.log('Found cover at:', self.cover_url)
+
+        # Find date
+        div = soup.find(id='page-title')
+        if div is not None:
+            h5 = div.find('h5')
+            if h5 is not None:
+                text = self.tag_to_string(h5)
+                date = text.partition(u'\u2022')[0].strip()
+                self.timefmt = u' [%s]'%date
+                self.log('Issue date:', date)
+
+        # Find TOC
+        toc = soup.find('ul', attrs={'class':'issue-article-list'})
         articles = []
-        for tag in date.itersiblings():
-            if tag.tag == 'h4': break
-            if tag.tag == 'p':
-                if tag.get('class') == 'indented':
-                    articles[-1]['description'] += html.tostring(tag)
-                else:
-                    href = tag.xpath('descendant::a[@href]')[0].get('href')
-                    article = {
-                               'title': u''.join(tag.xpath('descendant::text()')),
-                               'date' : '',
-                               'url'  : 'http://www.nybooks.com'+href,
-                               'description': '',
-                               }
-                    articles.append(article)
+        for li in toc.findAll('li'):
+            h3 = li.find('h3')
+            title = self.tag_to_string(h3)
+            author = self.tag_to_string(li.find('h4'))
+            title = title + u' (%s)'%author
+            url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
+            desc = ''
+            for p in li.findAll('p'):
+                desc += self.tag_to_string(p)
+            self.log('Found article:', title)
+            self.log('\t', url)
+            self.log('\t', desc)
+            articles.append({'title':title, 'url':url, 'date':'',
+                'description':desc})
 
         return [('Current Issue', articles)]
 

resources/recipes/npr_music_blogs.recipe

@@ -0,0 +1,18 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class nprmusic(BasicNewsRecipe):
+    title = 'NPR Music Blogs'
+    __author__ = 'cix3'
+    timefmt = ' [%b %d, %Y]'
+    language = 'en'
+
+    oldest_article = 30
+    max_articles_per_feed = 100
+    no_stylesheets = True
+
+    remove_tags = [dict(name='div', attrs={'id':['logo', 'comments', 'related_objects', 'inset module', 'footer', 'strip_control', 'header', 'navigation']}), dict(name='hr'), dict(name='img')]
+
+    feeds = [
+        ('A Blog Supreme', 'http://www.npr.org/blogs/ablogsupreme/index.xml'),
+        ('All Songs Considered', 'http://www.npr.org/blogs/allsongs/index.xml'),
+        ('Monitor Mix', 'http://www.npr.org/blogs/monitormix/index.xml')]

resources/recipes/nspm.recipe

@@ -6,6 +6,7 @@ nspm.rs
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
 
 class Nspm(BasicNewsRecipe):
     title                 = 'Nova srpska politicka misao'
@@ -22,22 +23,24 @@ class Nspm(BasicNewsRecipe):
     language              = 'sr'
     publication_type      = 'magazine'    
     masthead_url          = 'http://www.nspm.rs/templates/jsn_epic_pro/images/logol.jpg'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+    extra_css             = ' @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: "Times New Roman", serif1, serif} .article_description{font-family: Arial, sans1, sans-serif} img{margin-top:0.5em; margin-bottom: 0.7em} .author{color: #990000; font-weight: bold} .author,.createdate{font-size: 0.9em} img{margin-top:0.5em; margin-bottom: 0.7em} '
 
     conversion_options = {
-                          'comment'          : description
-                        , 'tags'             : category
-                        , 'publisher'        : publisher
-                        , 'language'         : language
-                        , 'linearize_tables' : True
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
                         }
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     remove_tags        = [
-                            dict(name=['link','object','embed'])
+                            dict(name=['link','object','embed','script','meta'])
                            ,dict(name='td', attrs={'class':'buttonheading'})
                          ]
-    remove_tags_after = dict(attrs={'class':'article_separator'})
+    keep_only_tags = [
+                         dict(attrs={'class':['contentpagetitle','author','createdate']})
+                        ,dict(name='p')                        
+                     ]
     remove_attributes = ['width','height']
 
     def get_browser(self):
@@ -53,4 +56,17 @@ class Nspm(BasicNewsRecipe):
     def preprocess_html(self, soup):
         for item in soup.body.findAll(style=True):
             del item['style']
-        return self.adeify_images(soup)
+        att = soup.find('a',attrs={'class':'contentpagetitle'})
+        if att:
+           att.name = 'h1';
+           del att['href']
+        att2 = soup.find('td')
+        if att2:
+           att2.name = 'p';
+           del att['valign']
+        for pt in soup.findAll('img'):
+            brtag = Tag(soup,'br')
+            brtag2 = Tag(soup,'br')
+            pt.append(brtag)
+            pt.append(brtag2)           
+        return soup

resources/recipes/pescanik.recipe

@@ -6,6 +6,7 @@ pescanik.net
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import Tag
 
 class Pescanik(BasicNewsRecipe):
     title                 = 'Pescanik'
@@ -13,32 +14,34 @@ class Pescanik(BasicNewsRecipe):
     description           = 'Pescanik'
     publisher             = 'Pescanik'
     category              = 'news, politics, Serbia'
-    oldest_article        = 5
+    oldest_article        = 10
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf-8'
     language              = 'sr'
     publication_type      = 'newsportal'    
-    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,"Lucida Grande",Tahoma,Verdana,sans1,sans-serif} .contentheading{font-size: x-large; font-weight: bold} .small{font-size: small} .createdate{font-size: x-small; font-weight: bold}'
+    extra_css             = ' @font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,"Lucida Grande",Tahoma,Verdana,sans1,sans-serif} .contentheading{font-size: x-large; font-weight: bold} .small{font-size: small} .createdate{font-size: x-small; font-weight: bold} '
 
     conversion_options = {
-                          'comment'          : description
-                        , 'tags'             : category
-                        , 'publisher'        : publisher
-                        , 'language'         : language
-                        , 'linearize_tables' : True
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
                         }
 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    
+    remove_attributes = ['valign','colspan','width','height','align','alt']
+    
+    remove_tags = [dict(name=['object','link','meta','script'])]
 
-    remove_tags = [
-                     dict(name='td'  , attrs={'class':'buttonheading'})
-                    ,dict(name='span', attrs={'class':'article_seperator'})
-                    ,dict(name=['object','link','h4','ul'])
-                  ]
-
+    keep_only_tags = [
+                         dict(attrs={'class':['contentheading','small','createdate']})
+                        ,dict(name='td', attrs={'valign':'top','colspan':'2'})                        
+                     ]
+                  
     feeds       = [(u'Pescanik Online', u'http://www.pescanik.net/index.php?option=com_rd_rss&id=12')]
 
     def print_version(self, url):
@@ -46,4 +49,12 @@ class Pescanik(BasicNewsRecipe):
         return nurl + '&pop=1&page=0'
 
     def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+        st = soup.findAll('td')
+        for it in st:
+            it.name='p'
+        for pt in soup.findAll('img'):
+            brtag = Tag(soup,'br')
+            brtag2 = Tag(soup,'br')
+            pt.append(brtag)
+            pt.append(brtag2)
+        return soup

resources/recipes/seanhannity.recipe

@@ -0,0 +1,35 @@
+#!/usr/bin/env  python
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class SeanHannity(BasicNewsRecipe):
+    cover_url             = 'http://www.hannity.com/images/misc_logo.gif'
+    title          		    = u"Sean Hannity Show"
+    __author__ 	          = 'Rob Lammert - rob.lammert[at]gmail.com'
+    description		        = u"Articles from Sean Hannity's website, www.hannity.com"
+    oldest_article        = 7.0
+    max_articles_per_feed = 100
+    recursions            = 0
+    encoding              = 'utf8'
+    no_stylesheets        = True
+    remove_javascript     = True
+    #use_embedded_content  = False
+
+    remove_tags    = [
+      dict(name='div', attrs={'id':['header','navsprite','topminibarad','headline_bar','shadow','footer']}),
+      dict(name='div', attrs={'class':'rightcolumn'}),
+      dict(name='table', attrs={'id':'audiobox'}),
+      dict(name='a', attrs={'title':['Home','Shows','Guests','Photos']}),
+      dict(name='iframe')
+    ]
+
+    feeds          = [
+	     ('Content Feed', u'http://feeds.feedburner.com/TheSeanHannityShow-AllContent?format=xml')
+    ]
+
+
+
+  #def print_version(self, url):
+    #parts=url.split('/')
+    #return url.replace(url, 'http://www.hannity.com/show/' + parts[4] + '/' + parts[5] + '/' + parts[6] + '?mode=print')
+

resources/recipes/thewest_au.recipe

@@ -0,0 +1,63 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+thewest.com.au
+'''
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheWest(BasicNewsRecipe):
+    title                 = 'The West Australian'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Australia'
+    publisher             = 'thewest.com.au'
+    category              = 'news, politics, Australia'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en_AU'
+    remove_empty_feeds    = True
+    publication_type      = 'newspaper'
+    masthead_url          = 'http://l.yimg.com/ao/i/mp/properties/news/02/wan/img/wan-logo-h49.png'
+    extra_css             = ' .article{font-family: Arial,Helvetica,sans-serif } .image{font-size: x-small} '
+
+    preprocess_regexps = [
+       (re.compile(r'</title>.*?</head>', re.DOTALL|re.IGNORECASE),lambda match: '</title></head>')
+    ]
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags       = [
+                          dict(attrs={'class':['tools','lhs']})
+                         ,dict(attrs={'id'   :'tools-bottom'})
+                         ,dict(attrs={'href' :'http://twitter.com/thewest_com_au'})
+                        ]
+    keep_only_tags    = [dict(attrs={'class':'mod article'})]
+    remove_attributes = ['width','height']
+
+
+    feeds = [
+               (u'WA News'       , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/wa.xml'            )
+              ,(u'National'      , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/national.xml'      )
+              ,(u'World'         , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/world.xml'         )
+              ,(u'Offbeat'       , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/offbeat.xml'       )
+              ,(u'Business'      , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/business.xml'      )
+              ,(u'Sport'         , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/sport.xml'         )
+              ,(u'Entertainment' , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/entertainment.xml' )
+              ,(u'Travel'        , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/travel.xml'        )
+              ,(u'Life+Style'    , u'http://d.yimg.com/au.rss.news.yahoo.com/thewest/lifestyle.xml'     )
+            ]
+
+    def get_article_url(self, article):
+        return article.get('guid',  None)
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

resources/recipes/usatoday.recipe

@@ -377,8 +377,9 @@ class USAToday(BasicNewsRecipe):
             if byline:
                 byline['class'] = 'byline'
                 # Replace comma with middot
-                byline.contents[0].replaceWith(re.sub(","," ·", byline.renderContents()))
-                return byline.renderContents()
+                byline.contents[0].replaceWith(re.sub(u",", u" ·",
+                    byline.renderContents(encoding=None)))
+                return byline.renderContents(encoding=None)
             else :
                 paras = soup.findAll(text=True)
                 for para in paras:

resources/recipes/virtualshackles.recipe

@@ -0,0 +1,33 @@
+__license__   = 'GPL v3'
+__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+'''
+www.virtualshackles.com
+'''
+
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class Virtualshackles(BasicNewsRecipe):
+    title                  = 'Virtual Shackles'
+    __author__             = 'Darko Miletic'
+    description            = "The adventures of Orion and Jack, making games they'd never play for people they don't like."
+    category               = 'virtual shackles, virtualshackles, games, webcomic, comic, video game, orion, jack'
+    oldest_article         = 10
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    use_embedded_content   = True
+    encoding               = 'cp1252'
+    publisher              = 'Virtual Shackles'
+    language               = 'en'
+    publication_type       = 'comic'
+
+    conversion_options = {
+                             'comments'  : description
+                            ,'tags'      : category
+                            ,'language'  : language
+                            ,'publisher' : publisher
+                         }
+
+    feeds = [(u'Virtual Shackles', u'http://feeds2.feedburner.com/virtualshackles' )]
+
+    def preprocess_html(self, soup):
+        return self.adeify_images(soup)

setup/check.py

@@ -106,6 +106,7 @@ class Check(Command):
         if os.path.exists(wn_path):
             sys.path.insert(0, wn_path)
             self.info('\tChecking Changelog...')
+            os.environ['DJANGO_SETTINGS_MODULE'] = 'calibre_servers.status.settings'
             import whats_new
             whats_new.test()
             sys.path.remove(wn_path)

src/calibre/__init__.py

@@ -7,6 +7,7 @@ import sys, os, re, logging, time, mimetypes, \
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
+from functools import partial
 
 warnings.simplefilter('ignore', DeprecationWarning)
 
@@ -399,42 +400,59 @@ def my_unichr(num):
     except ValueError:
         return u'?'
 
-def entity_to_unicode(match, exceptions=[], encoding='cp1252'):
+def entity_to_unicode(match, exceptions=[], encoding='cp1252',
+        result_exceptions={}):
     '''
-    @param match: A match object such that '&'+match.group(1)';' is the entity.
-    @param exceptions: A list of entities to not convert (Each entry is the name of the entity, for e.g. 'apos' or '#1234'
-    @param encoding: The encoding to use to decode numeric entities between 128 and 256.
+    :param match: A match object such that '&'+match.group(1)';' is the entity.
+
+    :param exceptions: A list of entities to not convert (Each entry is the name of the entity, for e.g. 'apos' or '#1234'
+
+    :param encoding: The encoding to use to decode numeric entities between 128 and 256.
     If None, the Unicode UCS encoding is used. A common encoding is cp1252.
+
+    :param result_exceptions: A mapping of characters to entities. If the result
+    is in result_exceptions, result_exception[result] is returned instead.
+    Convenient way to specify exception for things like < or > that can be
+    specified by various actual entities.
     '''
+    def check(ch):
+        return result_exceptions.get(ch, ch)
+
     ent = match.group(1)
     if ent in exceptions:
         return '&'+ent+';'
     if ent == 'apos':
-        return "'"
+        return check("'")
     if ent == 'hellips':
         ent = 'hellip'
-    if ent.startswith(u'#x'):
+    if ent.lower().startswith(u'#x'):
         num = int(ent[2:], 16)
         if encoding is None or num > 255:
-            return my_unichr(num)
-        return chr(num).decode(encoding)
+            return check(my_unichr(num))
+        return check(chr(num).decode(encoding))
     if ent.startswith(u'#'):
         try:
             num = int(ent[1:])
         except ValueError:
             return '&'+ent+';'
         if encoding is None or num > 255:
-            return my_unichr(num)
+            return check(my_unichr(num))
         try:
-            return chr(num).decode(encoding)
+            return check(chr(num).decode(encoding))
         except UnicodeDecodeError:
-            return my_unichr(num)
+            return check(my_unichr(num))
     try:
-        return my_unichr(name2codepoint[ent])
+        return check(my_unichr(name2codepoint[ent]))
     except KeyError:
         return '&'+ent+';'
 
 _ent_pat = re.compile(r'&(\S+?);')
+xml_entity_to_unicode = partial(entity_to_unicode, result_exceptions = {
+    '"' : '&quot;',
+    "'" : '&apos;',
+    '<' : '&lt;',
+    '>' : '&gt;',
+    '&' : '&amp;'})
 
 def prepare_string_for_xml(raw, attribute=False):
     raw = _ent_pat.sub(entity_to_unicode, raw)

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.6.47'
+__version__   = '0.6.49'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/customize/builtins.py

@@ -438,16 +438,17 @@ from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX
 from calibre.devices.nook.driver import NOOK
 from calibre.devices.prs500.driver import PRS500
 from calibre.devices.prs505.driver import PRS505, PRS700
-from calibre.devices.android.driver import ANDROID
+from calibre.devices.android.driver import ANDROID, S60
 from calibre.devices.nokia.driver import N770, N810
 from calibre.devices.eslick.driver import ESLICK
 from calibre.devices.nuut2.driver import NUUT2
 from calibre.devices.iriver.driver import IRIVER_STORY
 from calibre.devices.binatone.driver import README
-from calibre.devices.hanvon.driver import N516, EB511
+from calibre.devices.hanvon.driver import N516, EB511, ALEX
 from calibre.devices.edge.driver import EDGE
 from calibre.devices.teclast.driver import TECLAST_K3
 from calibre.devices.sne.driver import SNE
+from calibre.devices.misc import PALMPRE
 
 from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon
 from calibre.library.catalog import CSV_XML, EPUB_MOBI
@@ -506,6 +507,7 @@ plugins += [
     PRS700,
     PRS500,
     ANDROID,
+    S60,
     N770,
     N810,
     COOL_ER,
@@ -526,7 +528,9 @@ plugins += [
     ELONEX,
     TECLAST_K3,
     EDGE,
-    SNE
+    SNE,
+    ALEX,
+    PALMPRE
 ]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/customize/profiles.py

@@ -215,6 +215,7 @@ input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input,
         HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput,
         IRexDR1000Input, IRexDR800Input, NookInput]
 
+input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))
 
 class OutputProfile(Plugin):
 
@@ -239,6 +240,14 @@ class OutputProfile(Plugin):
     def tags_to_string(cls, tags):
         return ', '.join(tags)
 
+class iPadOutput(OutputProfile):
+
+    name = 'iPad'
+    short_name = 'ipad'
+    screen_size = (1024, 768)
+    comic_screen_size = (1024, 768)
+    dpi = 132.0
+
 class SonyReaderOutput(OutputProfile):
 
     name        = 'Sony Reader'
@@ -449,5 +458,8 @@ class NookOutput(OutputProfile):
 output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output,
         SonyReader900Output, MSReaderOutput, MobipocketOutput, HanlinV3Output,
         HanlinV5Output, CybookG3Output, CybookOpusOutput, KindleOutput,
+        iPadOutput,
         SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput,
-        IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput]
+        IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput,]
+
+output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower()))

src/calibre/devices/android/driver.py

@@ -56,3 +56,21 @@ class ANDROID(USBMS):
             dirs = [x.strip() for x in dirs.split(',')]
         self.EBOOK_DIR_MAIN = dirs
 
+class S60(USBMS):
+
+    name = 'S60 driver'
+    gui_name = 'S60 phone'
+    description    = _('Communicate with S60 phones.')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    VENDOR_ID = [0x421]
+    PRODUCT_ID = [0x156]
+    BCD = [0x100]
+
+    # For use with zxreader
+    FORMATS = ['fb2']
+    EBOOK_DIR_MAIN = 'FB2 Books'
+
+    VENDOR_NAME = 'NOKIA'
+    WINDOWS_MAIN_MEM = 'S60'

src/calibre/devices/hanvon/driver.py

@@ -34,6 +34,22 @@ class N516(USBMS):
     EBOOK_DIR_MAIN = 'e_book'
     SUPPORTS_SUB_DIRS = True
 
+class ALEX(N516):
+
+    name = 'Alex driver'
+    gui_name = 'SpringDesign Alex'
+    description    = _('Communicate with the SpringDesign Alex eBook reader.')
+    author         = 'Kovid Goyal'
+
+    FORMATS     = ['epub', 'pdf']
+    VENDOR_NAME      = 'ALEX'
+    WINDOWS_MAIN_MEM = 'READER'
+
+    MAIN_MEMORY_VOLUME_LABEL  = 'Alex Internal Memory'
+
+    EBOOK_DIR_MAIN = 'eBooks'
+    SUPPORTS_SUB_DIRS = True
+
 class EB511(USBMS):
     name           = 'Elonex EB 511 driver'
     gui_name       = 'EB 511'

src/calibre/devices/misc.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.devices.usbms.driver import USBMS
+
+class PALMPRE(USBMS):
+
+    name           = 'Palm Pre Device Interface'
+    gui_name       = 'Palm Pre'
+    description    = _('Communicate with the Palm Pre')
+    author         = 'Kovid Goyal'
+    supported_platforms = ['windows', 'osx', 'linux']
+
+    # Ordered list of supported formats
+    FORMATS     = ['mobi', 'prc', 'pdb', 'txt']
+
+    VENDOR_ID   = [0x0830]
+    PRODUCT_ID  = [0x8004, 0x8002]
+    BCD         = [0x0316]
+
+    VENDOR_NAME = 'PALM'
+    WINDOWS_MAIN_MEM = 'PRE'
+
+    EBOOK_DIR_MAIN = 'E-books'
+

src/calibre/ebooks/BeautifulSoup.py

@@ -1132,7 +1132,7 @@ class BeautifulStoneSoup(Tag, SGMLParser):
         self.pushTag(self)
 
     def popTag(self):
-        tag = self.tagStack.pop()
+        self.tagStack.pop()
         # Tags with just one string-owning child get the child as a
         # 'string' property, so that soup.tag.string is shorthand for
         # soup.tag.contents[0]
@@ -1727,7 +1727,7 @@ class UnicodeDammit:
             u = self._toUnicode(markup, proposed)
             self.markup = u
             self.originalEncoding = proposed
-        except Exception, e:
+        except Exception:
             #print "That didn't work!"
             #print e
             return None

src/calibre/ebooks/chardet/__init__.py

@@ -43,11 +43,8 @@ def strip_encoding_declarations(raw):
     return raw
 
 def substitute_entites(raw):
-    from calibre import entity_to_unicode
-    from functools import partial
-    f = partial(entity_to_unicode, exceptions=
-                ['amp', 'apos', 'quot', 'lt', 'gt'])
-    return ENTITY_PATTERN.sub(f, raw)
+    from calibre import xml_entity_to_unicode
+    return ENTITY_PATTERN.sub(xml_entity_to_unicode, raw)
 
 _CHARSET_ALIASES = { "macintosh" : "mac-roman",
                         "x-sjis" : "shift-jis" }

src/calibre/ebooks/comic/input.py

@@ -322,7 +322,7 @@ class ComicInput(InputFormatPlugin):
         ('margin_bottom', 0, OptionRecommendation.HIGH),
         ('insert_blank_line', False, OptionRecommendation.HIGH),
         ('remove_paragraph_spacing',  False, OptionRecommendation.HIGH),
-        ('dont_justify', True, OptionRecommendation.HIGH),
+        ('change_justification', 'left', OptionRecommendation.HIGH),
         ('dont_split_on_pagebreaks', True, OptionRecommendation.HIGH),
         ('chapter', None, OptionRecommendation.HIGH),
         ('page_breaks_brefore', None, OptionRecommendation.HIGH),

src/calibre/ebooks/conversion/cli.py

@@ -124,7 +124,7 @@ def add_pipeline_options(parser, plumber):
                       'linearize_tables',
                       'extra_css',
                       'margin_top', 'margin_left', 'margin_right',
-                      'margin_bottom', 'dont_justify',
+                      'margin_bottom', 'change_justification',
                       'insert_blank_line', 'remove_paragraph_spacing','remove_paragraph_spacing_indent_size',
                       'asciiize', 'remove_header', 'header_regex',
                       'remove_footer', 'footer_regex',

src/calibre/ebooks/conversion/plumber.py

@@ -299,12 +299,16 @@ OptionRecommendation(name='margin_right',
         help=_('Set the right margin in pts. Default is %default. '
             'Note: 72 pts equals 1 inch')),
 
-OptionRecommendation(name='dont_justify',
-        recommended_value=False, level=OptionRecommendation.LOW,
-        help=_('Do not force text to be justified in output. Whether text '
-            'is actually displayed justified or not depends on whether '
-            'the ebook format and reading device support justification.')
-        ),
+OptionRecommendation(name='change_justification',
+        recommended_value='original', level=OptionRecommendation.LOW,
+        choices=['left','justify','original'],
+        help=_('Change text justification. A value of "left" converts all'
+            ' justified text in the source to left aligned (i.e. '
+            'unjustified) text. A value of "justify" converts all '
+            'unjustified text to justified. A value of "original" '
+            '(the default) does not change justification in the '
+            'source file. Note that only some output formats support '
+            'justification.')),
 
 OptionRecommendation(name='remove_paragraph_spacing',
         recommended_value=False, level=OptionRecommendation.LOW,

src/calibre/ebooks/lrf/output.py

@@ -130,7 +130,7 @@ class LRFOutput(OutputFormatPlugin):
     ])
 
     recommendations = set([
-        ('dont_justify', True, OptionRecommendation.HIGH),
+        ('change_justification', 'original', OptionRecommendation.HIGH),
         ])
 
     def convert_images(self, pages, opts, wide):

src/calibre/ebooks/mobi/reader.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import functools, shutil, os, re, struct, textwrap, cStringIO, sys
+import shutil, os, re, struct, textwrap, cStringIO, sys
 
 try:
     from PIL import Image as PILImage
@@ -14,7 +14,7 @@ except ImportError:
 
 from lxml import html, etree
 
-from calibre import entity_to_unicode, CurrentDir
+from calibre import xml_entity_to_unicode, CurrentDir, entity_to_unicode
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import parse_date
 from calibre.ptempfile import TemporaryDirectory
@@ -302,9 +302,7 @@ class MobiReader(object):
 
         for pat in ENCODING_PATS:
             self.processed_html = pat.sub('', self.processed_html)
-        e2u = functools.partial(entity_to_unicode,
-            exceptions=['lt', 'gt', 'amp', 'apos', 'quot', '#60', '#62'])
-        self.processed_html = re.sub(r'&(\S+?);', e2u,
+        self.processed_html = re.sub(r'&(\S+?);', xml_entity_to_unicode,
             self.processed_html)
         self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
@@ -619,6 +617,7 @@ class MobiReader(object):
                 opf.cover = None
 
         cover = opf.cover
+        cover_copied = None
         if cover is not None:
             cover = cover.replace('/', os.sep)
             if os.path.exists(cover):
@@ -626,13 +625,19 @@ class MobiReader(object):
                 if os.path.exists(ncover):
                     os.remove(ncover)
                 shutil.copyfile(cover, ncover)
-            opf.cover = ncover.replace(os.sep, '/')
+                cover_copied = os.path.abspath(ncover)
+                opf.cover = ncover.replace(os.sep, '/')
 
         manifest = [(htmlfile, 'application/xhtml+xml'),
             (os.path.abspath('styles.css'), 'text/css')]
         bp = os.path.dirname(htmlfile)
+        added = set([])
         for i in getattr(self, 'image_names', []):
-            manifest.append((os.path.join(bp, 'images/', i), 'image/jpeg'))
+            path = os.path.join(bp, 'images', i)
+            added.add(path)
+            manifest.append((path, 'image/jpeg'))
+        if cover_copied is not None:
+            manifest.append((cover_copied, 'image/jpeg'))
 
         opf.create_manifest(manifest)
         opf.create_spine([os.path.basename(htmlfile)])

src/calibre/ebooks/mobi/writer.py

@@ -283,7 +283,7 @@ class Serializer(object):
                     if self.serialize_href(val, item):
                         continue
                 elif attr == 'src':
-                    href = item.abshref(val)
+                    href = urlnormalize(item.abshref(val))
                     if href in self.images:
                         index = self.images[href]
                         buffer.write('recindex="%05d"' % index)

src/calibre/ebooks/oeb/base.py

@@ -430,7 +430,10 @@ class DirContainer(object):
             return f.write(data)
 
     def exists(self, path):
-        path = os.path.join(self.rootdir, urlunquote(path))
+        try:
+            path = os.path.join(self.rootdir, urlunquote(path))
+        except ValueError: #Happens if path contains quoted special chars
+            return False
         return os.path.isfile(path)
 
     def namelist(self):
@@ -768,18 +771,12 @@ class Manifest(object):
                 % (self.id, self.href, self.media_type)
 
         def _parse_xml(self, data):
-            data = xml_to_unicode(data, strip_encoding_pats=True)[0]
+            data = xml_to_unicode(data, strip_encoding_pats=True,
+                    assume_utf8=True, resolve_entities=True)[0]
             if not data:
                 return None
             parser = etree.XMLParser(recover=True)
-            try:
-                return etree.fromstring(data, parser=parser)
-            except etree.XMLSyntaxError, err:
-                if getattr(err, 'code', 0) == 26 or str(err).startswith('Entity'):
-                    data = xml_to_unicode(data, strip_encoding_pats=True,
-                            resolve_entities=True)[0]
-                    return etree.fromstring(data)
-                raise
+            return etree.fromstring(data, parser=parser)
 
         def _parse_xhtml(self, data):
             self.oeb.log.debug('Parsing', self.href, '...')

src/calibre/ebooks/oeb/iterator.py

@@ -212,11 +212,12 @@ class EbookIterator(object):
 
         cover = self.opf.cover
         if self.ebook_ext in ('lit', 'mobi', 'prc', 'opf') and cover:
-            cfile = os.path.join(os.path.dirname(self.spine[0]),
-                    'calibre_iterator_cover.html')
-            chtml = (TITLEPAGE%cover).encode('utf-8')
+            cfile = os.path.join(self.base, 'calibre_iterator_cover.html')
+            chtml = (TITLEPAGE%os.path.relpath(cover, self.base).replace(os.sep,
+                '/')).encode('utf-8')
             open(cfile, 'wb').write(chtml)
-            self.spine[0:0] = [SpineItem(cfile)]
+            self.spine[0:0] = [SpineItem(cfile,
+                mime_type='application/xhtml+xml')]
             self.delete_on_exit.append(cfile)
 
         if self.opf.path_to_html_toc is not None and \

src/calibre/ebooks/oeb/stylizer.py

@@ -318,8 +318,8 @@ class Stylizer(object):
         if text == 'inherit':
             style['text-align'] = 'inherit'
         else:
-            if text in ('left', 'justify'):
-                val = 'left' if self.opts.dont_justify else 'justify'
+            if text in ('left', 'justify') and self.opts.change_justification in ('left', 'justify'):
+                val = self.opts.change_justification
                 style['text-align'] = val
             else:
                 style['text-align'] = text

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -138,8 +138,8 @@ class CSSFlattener(object):
                     float(self.context.margin_left))
             bs.append('margin-right : %fpt'%\
                     float(self.context.margin_right))
-            bs.append('text-align: '+ \
-                    ('left' if self.context.dont_justify else 'justify'))
+            if self.context.change_justification != 'original':
+                bs.append('text-align: '+ self.context.change_justification)
             body.set('style', '; '.join(bs))
             stylizer = Stylizer(html, item.href, self.oeb, self.context, profile,
                     user_css=self.context.extra_css,

src/calibre/ebooks/oeb/transforms/split.py

@@ -115,7 +115,7 @@ class Split(object):
         for i, x in enumerate(page_breaks):
             x.set('id', x.get('id', 'calibre_pb_%d'%i))
             id = x.get('id')
-            page_breaks_.append((XPath('//*[@id="%s"]'%id),
+            page_breaks_.append((XPath('//*[@id=%r]'%id),
                 x.get('pb_before', False)))
             page_break_ids.append(id)
 

src/calibre/ebooks/pml/pmlml.py

@@ -86,7 +86,7 @@ class PMLMLizer(object):
         self.toc = {}
         for item in oeb_book.toc:
             href, mid, id = item.href.partition('#')
-            aid = self.get_anchor_id(href, id)
+            self.get_anchor_id(href, id)
             if not self.toc.get(href, None):
                 self.toc[href] = {}
             self.toc[href][id] = item.title

src/calibre/gui2/convert/look_and_feel.py

@@ -19,7 +19,7 @@ class LookAndFeelWidget(Widget, Ui_Form):
 
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent, 'look_and_feel',
-                ['dont_justify', 'extra_css', 'base_font_size',
+                ['change_justification', 'extra_css', 'base_font_size',
                     'font_size_mapping', 'line_height',
                     'linearize_tables',
                     'disable_font_rescaling', 'insert_blank_line',

src/calibre/gui2/convert/look_and_feel.ui

@@ -84,7 +84,7 @@
         <string>...</string>
        </property>
        <property name="icon">
-        <iconset resource="../../../../resources/images.qrc">
+        <iconset>
          <normaloff>:/images/wizard.svg</normaloff>:/images/wizard.svg</iconset>
        </property>
        <property name="iconSize">
@@ -181,21 +181,7 @@
      </item>
     </layout>
    </item>
-   <item row="6" column="0">
-    <widget class="QCheckBox" name="opt_insert_blank_line">
-     <property name="text">
-      <string>Insert &amp;blank line</string>
-     </property>
-    </widget>
-   </item>
    <item row="7" column="0">
-    <widget class="QCheckBox" name="opt_dont_justify">
-     <property name="text">
-      <string>No text &amp;justification</string>
-     </property>
-    </widget>
-   </item>
-   <item row="8" column="0">
     <widget class="QCheckBox" name="opt_linearize_tables">
      <property name="text">
       <string>&amp;Linearize tables</string>
@@ -221,6 +207,42 @@
      </layout>
     </widget>
    </item>
+   <item row="8" column="0">
+    <widget class="QCheckBox" name="opt_insert_blank_line">
+     <property name="text">
+      <string>Insert &amp;blank line</string>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>Text justification:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="2">
+    <widget class="QComboBox" name="opt_change_justification">
+     <property name="currentIndex">
+      <number>2</number>
+     </property>
+     <item>
+      <property name="text">
+       <string>justify</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>left</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>original</string>
+      </property>
+     </item>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/dialogs/book_info.py

@@ -3,9 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
-'''
-'''
-import textwrap, os
+import textwrap, os, re
 
 from PyQt4.QtCore import QCoreApplication, SIGNAL, QModelIndex, QUrl, QTimer, Qt
 from PyQt4.QtGui import QDialog, QPixmap, QGraphicsScene, QIcon, QDesktopServices
@@ -97,7 +95,12 @@ class BookInfo(QDialog, Ui_BookInfo):
         info = self.view.model().get_book_info(row)
         self.setWindowTitle(info[_('Title')])
         self.title.setText('<b>'+info.pop(_('Title')))
-        self.comments.setText('<div>%s</div>' % info.pop(_('Comments'), ''))
+        comments = info.pop(_('Comments'), '')
+        if re.search(r'<[a-zA-Z]+>', comments) is None:
+            lines = comments.splitlines()
+            lines = [x if x.strip() else '<br><br>' for x in lines]
+            comments = '\n'.join(lines)
+        self.comments.setText('<div>%s</div>' % comments)
         cdata = info.pop('cover', '')
         self.cover_pixmap = QPixmap.fromImage(cdata)
         self.resize_cover()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -19,6 +19,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
         self.setupUi(self)
         self.db = db
         self.ids = [db.id(r) for r in rows]
+        self.groupBox.setTitle(_('Editing meta information for %d books') %
+                len(rows))
         self.write_series = False
         self.changed = False
         QObject.connect(self.button_box, SIGNAL("accepted()"), self.sync)

src/calibre/gui2/dialogs/scheduler.py

@@ -29,6 +29,7 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.recipe_model.do_refresh()
 
         self.search = SearchBox2(self)
+        self.search.setMinimumContentsLength(25)
         self.search.initialize('scheduler_search_history')
         self.recipe_box.layout().insertWidget(0, self.search)
         self.connect(self.search, SIGNAL('search(PyQt_PyObject,PyQt_PyObject)'),

src/calibre/gui2/dialogs/scheduler.ui

@@ -14,7 +14,7 @@
    <string>Schedule news download</string>
   </property>
   <property name="windowIcon">
-   <iconset resource="../../../../resources/images.qrc">
+   <iconset>
     <normaloff>:/images/scheduler.svg</normaloff>:/images/scheduler.svg</iconset>
   </property>
   <layout class="QGridLayout" name="gridLayout">
@@ -79,7 +79,7 @@
           <x>0</x>
           <y>0</y>
           <width>375</width>
-          <height>500</height>
+          <height>502</height>
          </rect>
         </property>
         <layout class="QVBoxLayout" name="verticalLayout_5">
@@ -371,7 +371,7 @@
       <enum>Qt::Horizontal</enum>
      </property>
      <property name="standardButtons">
-      <set>QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Save</set>
      </property>
     </widget>
    </item>

src/calibre/gui2/library.py

@@ -7,9 +7,9 @@ from math import cos, sin, pi
 from contextlib import closing
 
 from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \
-                        QItemDelegate, QPainterPath, QLinearGradient, QBrush, \
-                        QPen, QStyle, QPainter, \
-                        QImage, QApplication, QMenu, \
+                        QPainterPath, QLinearGradient, QBrush, \
+                        QPen, QStyle, QPainter, QStyleOptionViewItemV4, \
+                        QImage, QMenu, \
                         QStyledItemDelegate, QCompleter
 from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, pyqtSignal, \
                          SIGNAL, QObject, QSize, QModelIndex, QDate
@@ -28,14 +28,15 @@ from calibre.ebooks.metadata import string_to_authors, fmt_sidx, \
 from calibre.utils.config import tweaks
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 
-class LibraryDelegate(QItemDelegate):
+class LibraryDelegate(QStyledItemDelegate):
     COLOR    = QColor("blue")
     SIZE     = 16
     PEN      = QPen(COLOR, 1, Qt.SolidLine, Qt.RoundCap, Qt.RoundJoin)
 
     def __init__(self, parent):
-        QItemDelegate.__init__(self, parent)
+        QStyledItemDelegate.__init__(self, parent)
         self._parent = parent
+        self.dummy = QModelIndex()
         self.star_path = QPainterPath()
         self.star_path.moveTo(90, 50)
         for i in range(1, 5):
@@ -54,6 +55,9 @@ class LibraryDelegate(QItemDelegate):
         return QSize(5*(self.SIZE), self.SIZE+4)
 
     def paint(self, painter, option, index):
+        style = self._parent.style()
+        option = QStyleOptionViewItemV4(option)
+        self.initStyleOption(option, self.dummy)
         num = index.model().data(index, Qt.DisplayRole).toInt()[0]
         def draw_star():
             painter.save()
@@ -66,11 +70,10 @@ class LibraryDelegate(QItemDelegate):
 
         painter.save()
         if hasattr(QStyle, 'CE_ItemViewItem'):
-            QApplication.style().drawControl(QStyle.CE_ItemViewItem, option,
+            style.drawControl(QStyle.CE_ItemViewItem, option,
                     painter, self._parent)
         elif option.state & QStyle.State_Selected:
             painter.fillRect(option.rect, option.palette.highlight())
-        self.drawFocus(painter, option, option.rect)
         try:
             painter.setRenderHint(QPainter.Antialiasing)
             painter.setClipRect(option.rect)
@@ -89,7 +92,7 @@ class LibraryDelegate(QItemDelegate):
         painter.restore()
 
     def createEditor(self, parent, option, index):
-        sb = QItemDelegate.createEditor(self, parent, option, index)
+        sb = QStyledItemDelegate.createEditor(self, parent, option, index)
         sb.setMinimum(0)
         sb.setMaximum(5)
         return sb
@@ -774,7 +777,7 @@ class BooksView(TableView):
             self.setItemDelegateForColumn(cm.index('series'), self.series_delegate)
 
     def set_context_menu(self, edit_metadata, send_to_device, convert, view,
-                         save, open_folder, book_details, delete, similar_menu=None):
+                         save, open_folder, book_details, merge, delete, similar_menu=None):
         self.setContextMenuPolicy(Qt.DefaultContextMenu)
         self.context_menu = QMenu(self)
         if edit_metadata is not None:
@@ -787,6 +790,8 @@ class BooksView(TableView):
         self.context_menu.addAction(save)
         if open_folder is not None:
             self.context_menu.addAction(open_folder)
+        if merge is not None:
+            self.context_menu.addAction(merge)
         if delete is not None:
             self.context_menu.addAction(delete)
         if book_details is not None:

src/calibre/gui2/main.ui

@@ -595,6 +595,21 @@
     <bool>false</bool>
    </property>
   </action>
+  <action name="action_merge">
+   <property name="icon">
+    <iconset resource="../../../resources/images.qrc">
+     <normaloff>:/images/merge_books.svg</normaloff>:/images/merge_books.svg</iconset>
+   </property>
+   <property name="text">
+    <string>Merge books</string>
+   </property>
+   <property name="shortcut">
+    <string>M</string>
+   </property>
+   <property name="autoRepeat">
+    <bool>false</bool>
+   </property>
+  </action>
   <action name="action_sync">
    <property name="enabled">
     <bool>false</bool>

src/calibre/gui2/ui.py

@@ -274,6 +274,16 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         md.addAction(_('Download only covers'))
         md.addAction(_('Download only social metadata'))
         self.metadata_menu = md
+
+        mb = QMenu()
+        mb.addAction(_('Merge into first selected book - delete others'))
+        mb.addSeparator()
+        mb.addAction(_('Merge into first selected book - keep others'))
+        self.merge_menu = mb
+        self.action_merge.setMenu(mb)
+        md.addSeparator()
+        md.addAction(self.action_merge)
+
         self.add_menu = QMenu()
         self.add_menu.addAction(_('Add books from a single directory'))
         self.add_menu.addAction(_('Add books from directories, including '
@@ -320,6 +330,13 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         QObject.connect(md.actions()[7], SIGNAL('triggered(bool)'),
                 self.__em6__)
 
+        QObject.connect(self.action_merge, SIGNAL("triggered(bool)"),
+                self.merge_books)
+        QObject.connect(mb.actions()[0], SIGNAL('triggered(bool)'),
+                self.merge_books)
+        self.__mb1__ = partial(self.merge_books, safe_merge=True)
+        QObject.connect(mb.actions()[2], SIGNAL('triggered(bool)'),
+                self.__mb1__)
 
         self.save_menu = QMenu()
         self.save_menu.addAction(_('Save to disk'))
@@ -362,6 +379,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 Qt.QueuedConnection)
         self.connect(self.action_open_containing_folder,
                 SIGNAL('triggered(bool)'), self.view_folder)
+
         self.delete_menu.actions()[0].triggered.connect(self.delete_books)
         self.delete_menu.actions()[1].triggered.connect(self.delete_selected_formats)
         self.delete_menu.actions()[2].triggered.connect(self.delete_all_but_selected_formats)
@@ -374,6 +392,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.create_device_menu()
         self.action_edit.setMenu(md)
         self.action_save.setMenu(self.save_menu)
+
         cm = QMenu()
         cm.addAction(_('Convert individually'))
         cm.addAction(_('Bulk convert'))
@@ -459,14 +478,17 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                                         self.action_save,
                                         self.action_open_containing_folder,
                                         self.action_show_book_details,
+                                        self.action_merge,
                                         self.action_del,
                                         similar_menu=similar_menu)
+
         self.memory_view.set_context_menu(None, None, None,
-                self.action_view, self.action_save, None, None, self.action_del)
+                self.action_view, self.action_save, None, None, None, self.action_del)
         self.card_a_view.set_context_menu(None, None, None,
-                self.action_view, self.action_save, None, None, self.action_del)
+                self.action_view, self.action_save, None, None, None, self.action_del)
         self.card_b_view.set_context_menu(None, None, None,
-                self.action_view, self.action_save, None, None, self.action_del)
+                self.action_view, self.action_save, None, None, None, self.action_del)
+
         QObject.connect(self.library_view,
                 SIGNAL('files_dropped(PyQt_PyObject)'),
                         self.files_dropped, Qt.QueuedConnection)
@@ -851,7 +873,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         else:
             print msg
 
-
     def current_view(self):
         '''Convenience method that returns the currently visible view '''
         idx = self.stack.currentIndex()
@@ -1587,6 +1608,127 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
 
     ############################################################################
 
+    ############################### Merge books ##############################
+    def merge_books(self, safe_merge=False):
+        '''
+        Merge selected books in library.
+        '''
+        if self.stack.currentIndex() != 0:
+            return
+        rows = self.library_view.selectionModel().selectedRows()
+        if not rows or len(rows) == 0:
+            return error_dialog(self, _('Cannot merge books'),
+                                _('No books selected'), show=True)
+        if len(rows) < 2:
+            return error_dialog(self, _('Cannot merge books'),
+                        _('At least two books must be selected for merging'),
+                        show=True)
+        dest_id, src_books, src_ids = self.books_to_merge(rows)
+        if safe_merge:
+            if not confirm('<p>'+_(
+                'All book formats and metadata from the selected books '
+                'will be added to the <b>first selected book.</b><br><br> '
+                'The second and subsequently selected books will not '
+                'be deleted or changed.<br><br>'
+                'Please confirm you want to proceed.')
+            +'</p>', 'merge_books_safe', self):
+                return
+            self.add_formats(dest_id, src_books)
+            self.merge_metadata(dest_id, src_ids)
+        else:
+            if not confirm('<p>'+_(
+                'All book formats and metadata from the selected books will be merged '
+                'into the <b>first selected book</b>.<br><br>'
+                'After merger the second and '
+                'subsequently selected books will be <b>deleted</b>. <br><br>'
+                'All book formats of the first selected book will be kept '
+                'and any duplicate formats in the second and subsequently selected books '
+                'will be permanently <b>deleted</b> from your computer.<br><br>  '
+                'Are you <b>sure</b> you want to proceed?')
+            +'</p>', 'merge_books', self):
+                return
+            if len(rows)>5:
+                if not confirm('<p>'+_('You are about to merge more than 5 books.  '
+                                        'Are you <b>sure</b> you want to proceed?')
+                                    +'</p>', 'merge_too_many_books', self):
+                    return
+            self.add_formats(dest_id, src_books)
+            self.merge_metadata(dest_id, src_ids)
+            self.delete_books_after_merge(src_ids)
+            # leave the selection highlight on first selected book
+            dest_row = rows[0].row()
+            for row in rows:
+                if row.row() < rows[0].row():
+                    dest_row -= 1
+            ci = self.library_view.model().index(dest_row, 0)
+            if ci.isValid():
+                self.library_view.setCurrentIndex(ci)
+
+    def add_formats(self, dest_id, src_books, replace=False):
+        for src_book in src_books:
+            if src_book:
+                fmt = os.path.splitext(src_book)[-1].replace('.', '').upper()
+                with open(src_book, 'rb') as f:
+                    self.db.add_format(dest_id, fmt, f, index_is_id=True,
+                            notify=False, replace=replace)
+
+    def books_to_merge(self, rows):
+        src_books = []
+        src_ids = []
+        m = self.library_view.model()
+        for i, row in enumerate(rows):
+            id_ = m.id(row)
+            if i == 0:
+                dest_id = id_
+            else:
+                src_ids.append(id_)
+                dbfmts = m.db.formats(id_, index_is_id=True)
+                if dbfmts:
+                    for fmt in dbfmts:
+                        src_books.append(m.db.format_abspath(id_, fmt,
+                            index_is_id=True))
+        return [dest_id, src_books, src_ids]
+
+    def delete_books_after_merge(self, ids_to_delete):
+        self.library_view.model().delete_books_by_id(ids_to_delete)
+
+    def merge_metadata(self, dest_id, src_ids):
+        db = self.library_view.model().db
+        dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True)
+        orig_dest_comments = dest_mi.comments
+        for src_id in src_ids:
+            src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True)
+            if src_mi.comments and orig_dest_comments != src_mi.comments:
+                if not dest_mi.comments or len(dest_mi.comments) == 0:
+                    dest_mi.comments = src_mi.comments
+                else:
+                    dest_mi.comments = unicode(dest_mi.comments) + u'\n\n' + unicode(src_mi.comments)
+            if src_mi.title and src_mi.title and (not dest_mi.title or
+                    dest_mi.title == _('Unknown')):
+                dest_mi.title = src_mi.title
+            if src_mi.title and (not dest_mi.authors or dest_mi.authors[0] ==
+                    _('Unknown')):
+                dest_mi.authors = src_mi.authors
+                dest_mi.author_sort = src_mi.author_sort
+            if src_mi.tags:
+                if not dest_mi.tags:
+                    dest_mi.tags = src_mi.tags
+                else:
+                    for tag in src_mi.tags:
+                        dest_mi.tags.append(tag)
+            if src_mi.cover and not dest_mi.cover:
+                dest_mi.cover = src_mi.cover
+            if not dest_mi.publisher:
+                dest_mi.publisher = src_mi.publisher
+            if not dest_mi.rating:
+                dest_mi.rating = src_mi.rating
+            if not dest_mi.series:
+                dest_mi.series = src_mi.series
+                dest_mi.series_index = src_mi.series_index
+        db.set_metadata(dest_id, dest_mi, ignore_errors=False)
+
+    ############################################################################
+
 
     ############################## Save to disk ################################
     def save_single_format_to_disk(self, checked):
@@ -2096,6 +2238,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self.status_bar.reset_info()
         if location == 'library':
             self.action_edit.setEnabled(True)
+            self.action_merge.setEnabled(True)
             self.action_convert.setEnabled(True)
             self.view_menu.actions()[1].setEnabled(True)
             self.action_open_containing_folder.setEnabled(True)
@@ -2106,6 +2249,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 action.setEnabled(True)
         else:
             self.action_edit.setEnabled(False)
+            self.action_merge.setEnabled(False)
             self.action_convert.setEnabled(False)
             self.view_menu.actions()[1].setEnabled(False)
             self.action_open_containing_folder.setEnabled(False)

src/calibre/gui2/viewer/main.py

@@ -194,6 +194,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.tool_bar2.insertSeparator(self.action_find_next)
         self.setFocusPolicy(Qt.StrongFocus)
         self.search = SearchBox2(self)
+        self.search.setMinimumContentsLength(20)
         self.search.initialize('viewer_search_history')
         self.search.setToolTip(_('Search for text in book'))
         self.search.setMinimumWidth(200)

src/calibre/gui2/widgets.py

@@ -551,7 +551,8 @@ class LineEditECM(object):
         self.setText(unicode(self.text()).swapcase())
 
     def title_case(self):
-        self.setText(unicode(self.text()).title())
+        from calibre.utils.titlecase import titlecase
+        self.setText(titlecase(unicode(self.text())))
 
 
 class EnLineEdit(LineEditECM, QLineEdit):

src/calibre/library/database2.py

@@ -123,10 +123,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.is_case_sensitive = not iswindows and not isosx and \
             not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
         SchemaUpgrade.__init__(self)
-        CustomColumns.__init__(self)
         self.initialize_dynamic()
 
     def initialize_dynamic(self):
+        CustomColumns.__init__(self)
         template = '''\
                 (SELECT {query} FROM books_{table}_link AS link INNER JOIN
                     {table} ON(link.{link_col}={table}.id) WHERE link.book=books.id)
@@ -1428,6 +1428,7 @@ books_series_link      feeds
             os.remove(self.dbpath)
             shutil.copyfile(dest, self.dbpath)
             self.connect()
+            self.initialize_dynamic()
             self.refresh()
         if os.path.exists(dest):
             os.remove(dest)

src/calibre/library/schema_upgrades.py

@@ -266,6 +266,6 @@ class SchemaUpgrade(object):
                     UNIQUE(label)
                 );
                 CREATE INDEX custom_columns_idx ON custom_columns (label);
-                CREATE INDEX formats_idx ON data (format);
+                CREATE INDEX IF NOT EXISTS formats_idx ON data (format);
         ''')
 

src/calibre/manual/faq.rst

@@ -81,7 +81,7 @@ Device Integration
 
 What devices does |app| support?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-At the moment |app| has full support for the SONY PRS 300/500/505/600/700/900, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle 1/2/DX, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3, various Android phones and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
+At the moment |app| has full support for the SONY PRS 300/500/505/600/700/900, Barnes & Noble Nook, Cybook Gen 3/Opus, Amazon Kindle 1/2/DX, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook 360, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3, SpringDesign Alex, various Android phones and the iPhone. In addition, using the :guilabel:`Save to disk` function you can use it with any ebook reader that exports itself as a USB disk.
 
 How can I help get my device supported in |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/utils/titlecase.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
+Python version by Stuart Colville http://muffinresearch.co.uk
+License: http://www.opensource.org/licenses/mit-license.php
+"""
+
+import re
+
+__all__ = ['titlecase']
+__version__ = '0.5'
+
+SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
+PUNCT = r"""!"#$%&'‘()*+,\-./:;?@[\\\]_`{|}~"""
+
+SMALL_WORDS = re.compile(r'^(%s)$' % SMALL, re.I)
+INLINE_PERIOD = re.compile(r'[a-z][.][a-z]', re.I)
+UC_ELSEWHERE = re.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT)
+CAPFIRST = re.compile(r"^[%s]*?([A-Za-z])" % PUNCT)
+SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), re.I)
+SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), re.I)
+SUBPHRASE = re.compile(r'([:.;?!][ ])(%s)' % SMALL)
+APOS_SECOND = re.compile(r"^[dol]{1}['‘]{1}[a-z]+$", re.I)
+ALL_CAPS = re.compile(r'^[A-Z\s%s]+$' % PUNCT)
+UC_INITIALS = re.compile(r"^(?:[A-Z]{1}\.{1}|[A-Z]{1}\.{1}[A-Z]{1})+$")
+MAC_MC = re.compile(r"^([Mm]a?c)(\w+)")
+
+def titlecase(text):
+
+    """
+    Titlecases input text
+
+    This filter changes all words to Title Caps, and attempts to be clever
+    about *un*capitalizing SMALL words like a/an/the in the input.
+
+    The list of "SMALL words" which are not capped comes from
+    the New York Times Manual of Style, plus 'vs' and 'v'.
+
+    """
+
+    all_caps = ALL_CAPS.match(text)
+
+    words = re.split('\s', text)
+    line = []
+    for word in words:
+        if all_caps:
+            if UC_INITIALS.match(word):
+                line.append(word)
+                continue
+            else:
+                word = word.lower()
+
+        if APOS_SECOND.match(word):
+            word = word.replace(word[0], word[0].upper())
+            word = word.replace(word[2], word[2].upper())
+            line.append(word)
+            continue
+        if INLINE_PERIOD.search(word) or UC_ELSEWHERE.match(word):
+            line.append(word)
+            continue
+        if SMALL_WORDS.match(word):
+            line.append(word.lower())
+            continue
+
+        match = MAC_MC.match(word)
+        if match:
+            line.append("%s%s" % (match.group(1).capitalize(),
+                                  match.group(2).capitalize()))
+            continue
+
+        hyphenated = []
+        for item in word.split('-'):
+            hyphenated.append(CAPFIRST.sub(lambda m: m.group(0).upper(), item))
+        line.append("-".join(hyphenated))
+
+
+    result = " ".join(line)
+
+    result = SMALL_FIRST.sub(lambda m: '%s%s' % (
+        m.group(1),
+        m.group(2).capitalize()
+    ), result)
+
+    result = SMALL_LAST.sub(lambda m: m.group(0).capitalize(), result)
+
+    result = SUBPHRASE.sub(lambda m: '%s%s' % (
+        m.group(1),
+        m.group(2).capitalize()
+    ), result)
+
+    return result
+

src/calibre/web/feeds/recipes/model.py

@@ -113,7 +113,7 @@ class NewsItem(NewsTreeItem):
         return NONE
 
     def __cmp__(self, other):
-        return cmp(self.title, getattr(other, 'title', ''))
+        return cmp(self.title.lower(), getattr(other, 'title', '').lower())
 
 class RecipeModel(QAbstractItemModel, SearchQueryParser):