Merge from trunk

Changelog.yaml

@@ -19,6 +19,58 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.10
+  date: 2011-07-15
+
+  new features:
+    - title: "Add a right click menu to the cover browser. It allows you to view a book, edit metadata etc. from within the cover browser. The menu can be customized in Preferences->Toolbars"
+
+    - title: "Allow selecting and stopping multiple jobs at once in the jobs window"
+      tickets: [810349]
+
+    - title: "When editing metadata directly in the book list, have a little pop up menu so that all existing values can be accessed by mouse only. For example, when you edit authors, you can use the mouse to select an existing author."
+
+    - title: "Get Books: Add ebook.nl and fix price parsing for the legimi store"
+
+    - title: "Drivers for Samsung Infuse and Motorola XPERT"
+
+    - title: "Tag Browser: Make hierarchical items work in group searched terms."
+
+  bug fixes:
+    - title: "Allow setting numbers larger than 99 in custom series columns"
+
+    - title: "Fix a bug that caused the same news download sent via a USB connection to the device on two different days resulting in a duplicate on the device"
+
+    - title: "Ensure English in the list of interface languages in Preferences is always listed in English, so that it does not become hard to find"
+
+    - title: "SNB Output: Fix bug in handling unicode file names"
+
+    - title: "Fix sorting problem in manage categories. Fix poor performance problem when dropping multiple books onto a user category."
+
+    - title: "Remove 'empty field' error dialogs in bulk search/replace, instead setting the fields to their default value."
+
+    - title: "Fix regression that broke communicating with Kobo devices using outdated firmware"
+      tickets: [807832]
+
+    - title: "LRF Input: Fix conversion of LRF files with non ascii titles on some windows systems"
+      tickets: [807641]
+
+  improved recipes:
+    - Time
+    - Freakonomics Blog
+    - io9
+    - "Computer Act!ve"
+
+  new recipes:
+    - title: Techcrunch and Pecat
+      author: Darko Miletic
+
+    - title: Vio Mundo, IDG Now and Tojolaco
+      author: Diniz Bortoletto
+
+    - title: Geek and Poke, Automatiseringgids IT 
+      author: DrMerry
+
 - version: 0.8.9
   date: 2011-07-08
 
@@ -32,7 +84,7 @@
     - title: "Conversion pipeline: Add option to control if duplicate entries are allowed when generating the Table of Contents from links."
       tickets: [806095]
 
-    - title: "Metadata download: When merging results, if the query to the xisbn service hangs, wait no more than 10 seconds. Also try harder to preserve the month when downlaoding published date. Do not throw away isbnless results if there are some sources that return isbns and some that do not."
+    - title: "Metadata download: When merging results, if the query to the xisbn service hangs, wait no more than 10 seconds. Also try harder to preserve the month when downloading published date. Do not throw away isbnless results if there are some sources that return isbns and some that do not."
       tickets: [798309]
 
     - title: "Get Books: Remove OpenLibrary since it has the same files as archive.org. Allow direct downloading from Project Gutenberg."

recipes/freakonomics.recipe

@@ -1,25 +1,29 @@
 #!/usr/bin/env  python
 __license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__copyright__ = '2011, Starson17'
 __docformat__ = 'restructuredtext en'
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Freakonomics(BasicNewsRecipe):
-
     title = 'Freakonomics Blog'
     description = 'The Hidden side of everything'
     __author__          = 'Starson17'
+    __version__         = '1.02'
+    __date__            = '11 July 2011'
     language = 'en'
     cover_url           = 'http://ilkerugur.files.wordpress.com/2009/04/freakonomics.jpg'
+    use_embedded_content= False
+    no_stylesheets      = True
+    oldest_article      = 30
+    remove_javascript   = True
+    remove_empty_feeds    = True
+    max_articles_per_feed = 50
 
-    feeds = [('Blog', 'http://feeds.feedburner.com/freakonomicsblog')]
-
-    keep_only_tags     = [dict(name='div', attrs={'id':'header'}),
-                          dict(name='h1'),
-                          dict(name='h2'),
-                          dict(name='div', attrs={'class':'entry-content'}),
-                          ]
+    feeds = [(u'Freakonomics Blog', u'http://www.freakonomics.com/feed/')]
+    keep_only_tags = [dict(name='div', attrs={'id':['content']})]
+    remove_tags_after = [dict(name='div', attrs={'class':['simple_socialmedia']})]
+    remove_tags = [dict(name='div', attrs={'class':['simple_socialmedia','single-fb-share','wp-polls']})]
     extra_css = '''
                     h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
                     h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}

recipes/instapaper.recipe

@@ -1,4 +1,3 @@
-from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1299694372(BasicNewsRecipe):
@@ -9,14 +8,24 @@ class AdvancedUserRecipe1299694372(BasicNewsRecipe):
     oldest_article               = 365
     max_articles_per_feed = 100
     no_stylesheets        = True
+    remove_javascript     = True
+    remove_tags              = [
+	dict(name='div', attrs={'id':'text_controls_toggle'})
+	,dict(name='script')
+	,dict(name='div', attrs={'id':'text_controls'})
+	,dict(name='div', attrs={'id':'editing_controls'})
+	,dict(name='div', attrs={'class':'bar bottom'})
+	 ]
     use_embedded_content  = False
     needs_subscription    = True
     INDEX                 = u'http://www.instapaper.com'
     LOGIN                 = INDEX + u'/user/login'
 
 
-
-    feeds          = [(u'Instapaper Unread', u'http://www.instapaper.com/u'), (u'Instapaper Starred', u'http://www.instapaper.com/starred')]
+    feeds          = [
+            (u'Instapaper Unread', u'http://www.instapaper.com/u'),
+            (u'Instapaper Starred', u'http://www.instapaper.com/starred')
+            ]
 
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
@@ -37,18 +46,20 @@ class AdvancedUserRecipe1299694372(BasicNewsRecipe):
             self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
             articles = []
             soup = self.index_to_soup(feedurl)
-            for item in soup.findAll('div', attrs={'class':'titleRow'}):
-                description = self.tag_to_string(item.div)
+            for item in soup.findAll('div', attrs={'class':'cornerControls'}):
+                #description = self.tag_to_string(item.div)
                 atag = item.a
                 if atag and atag.has_key('href'):
                     url         = atag['href']
-                    title       = self.tag_to_string(atag)
-                    date        = strftime(self.timefmt)
                     articles.append({
-                                      'title'      :title
-                                     ,'date'       :date
-                                     ,'url'        :url
-                                     ,'description':description
+                                     'url'        :url
                                     })
             totalfeeds.append((feedtitle, articles))
         return totalfeeds
+
+    def print_version(self, url):
+        return 'http://www.instapaper.com' + url
+
+    def populate_article_metadata(self, article, soup, first):
+        article.title  = soup.find('title').contents[0].strip()
+

recipes/io9.recipe

@@ -16,16 +16,14 @@ class i09(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = True
     encoding              = 'utf-8'
-    use_embedded_content  = False
+    use_embedded_content  = True
     language              = 'en'
     masthead_url          = 'http://cache.gawkerassets.com/assets/io9.com/img/logo.png'
     extra_css             = '''
 	body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif}
 	img{margin-bottom: 1em}
 	h1{font-family :Arial,Helvetica,sans-serif; font-size:large}
-	h2{font-family :Arial,Helvetica,sans-serif; font-size:x-small}
 	              '''
-
     conversion_options = {
                           'comment'   : description
                         , 'tags'      : category
@@ -33,13 +31,11 @@ class i09(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    remove_attributes  = ['width','height']
-    keep_only_tags     = [dict(attrs={'class':'content permalink'})]
-    remove_tags_before = dict(name='h1')
-    remove_tags        = [dict(attrs={'class':'contactinfo'})]
-    remove_tags_after  = dict(attrs={'class':'contactinfo'})
+    feeds = [(u'Articles', u'http://feeds.gawker.com/io9/vip?format=xml')]
 
-    feeds = [(u'Articles', u'http://feeds.gawker.com/io9/full')]
+    remove_tags = [
+            {'class': 'feedflare'},
+    ]
 
     def preprocess_html(self, soup):
         return self.adeify_images(soup)

recipes/nikkei_news.recipe

@@ -0,0 +1,88 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+import re
+
+#import pprint, sys
+#pp = pprint.PrettyPrinter(indent=4)
+
+class NikkeiNet_paper_subscription(BasicNewsRecipe):
+    title           = u'\u65E5\u672C\u7D4C\u6E08\u65B0\u805E\uFF08\u671D\u520A\u30FB\u5915\u520A\uFF09'
+    __author__      = 'Ado Nishimura'
+    description     = u'\u65E5\u7D4C\u96FB\u5B50\u7248\u306B\u3088\u308B\u65E5\u672C\u7D4C\u6E08\u65B0\u805E\u3002\u671D\u520A\u30FB\u5915\u520A\u306F\u53D6\u5F97\u6642\u9593\u306B\u3088\u308A\u5207\u308A\u66FF\u308F\u308A\u307E\u3059\u3002\u8981\u8CFC\u8AAD'
+    needs_subscription = True
+    oldest_article  = 1
+    max_articles_per_feed = 30
+    language        = 'ja'
+    no_stylesheets  = True
+    cover_url       = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
+    masthead_url    = 'http://parts.nikkei.com/parts/ds/images/common/logo_r1.svg'
+
+    remove_tags_before = {'class':"cmn-indent"}
+    remove_tags = [
+#                       {'class':"cmn-article_move"},
+#                       {'class':"cmn-pr_list"},
+#                       {'class':"cmnc-zoom"},
+                       {'class':"cmn-hide"},
+                       {'name':'form'},
+                  ]
+    remove_tags_after = {'class':"cmn-indent"}
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+
+        #pp.pprint(self.parse_index())
+        #exit(1)
+
+        #br.set_debug_http(True)
+        #br.set_debug_redirects(True)
+        #br.set_debug_responses(True)
+
+        if self.username is not None and self.password is not None:
+            print "----------------------------open top page----------------------------------------"
+            br.open('http://www.nikkei.com/')
+            print "----------------------------open first login form--------------------------------"
+            link = br.links(url_regex="www.nikkei.com/etc/accounts/login").next()
+            br.follow_link(link)
+            #response = br.response()
+            #print response.get_data()
+            print "----------------------------JS redirect(send autoPostForm)-----------------------"
+            br.select_form(name='autoPostForm')
+            br.submit()
+            #response = br.response()
+            print "----------------------------got login form---------------------------------------"
+            br.select_form(name='LA0210Form01')
+            br['LA0210Form01:LA0210Email']    = self.username
+            br['LA0210Form01:LA0210Password'] = self.password
+            br.submit()
+            #response = br.response()
+            print "----------------------------JS redirect------------------------------------------"
+            br.select_form(nr=0)
+            br.submit()
+
+            #br.set_debug_http(False)
+            #br.set_debug_redirects(False)
+            #br.set_debug_responses(False)
+        return br
+
+    def cleanup(self):
+        print "----------------------------logout-----------------------------------------------"
+        self.browser.open('https://regist.nikkei.com/ds/etc/accounts/logout')
+
+    def parse_index(self):
+        print "----------------------------get index of paper-----------------------------------"
+        result = []
+        soup = self.index_to_soup('http://www.nikkei.com/paper/')
+        #soup = self.index_to_soup(self.test_data())
+        for sect in soup.findAll('div', 'cmn-section kn-special JSID_baseSection'):
+            sect_title = sect.find('h3', 'cmnc-title').string
+            sect_result = []
+            for elem in sect.findAll(attrs={'class':['cmn-article_title']}):
+                url = 'http://www.nikkei.com' + elem.span.a['href']
+                url = re.sub("/article/", "/print-article/", url) # print version.
+                span = elem.span.a.span
+                if ((span is not None) and (len(span.contents) > 1)):
+                    title = span.contents[1].string
+                    sect_result.append(dict(title=title, url=url, date='',
+                                            description='', content=''))
+            result.append([sect_title, sect_result])
+        #pp.pprint(result)
+

recipes/techcrunch.recipe

@@ -0,0 +1,63 @@
+__license__   = 'GPL v3'
+__copyright__ = '2011, Darko Miletic <darko.miletic at gmail.com>'
+'''
+techcrunch.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TechCrunch(BasicNewsRecipe):
+    title                 = 'TechCrunch'
+    __author__            = 'Darko Miletic'
+    description           = 'IT News'
+    publisher             = 'AOL Inc.'
+    category              = 'news, IT'
+    oldest_article        = 2
+    max_articles_per_feed = 200
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    publication_type      = 'newsportal'
+    masthead_url          = 'http://s2.wp.com/wp-content/themes/vip/tctechcrunch2/images/site-logo.png'
+    extra_css             = """
+                               body{font-family: Helvetica,Arial,sans-serif }
+                               img{margin-bottom: 0.4em; display:block}
+                            """
+
+    conversion_options = {
+                          'comment'   : description
+                        , 'tags'      : category
+                        , 'publisher' : publisher
+                        , 'language'  : language
+                        }
+
+    remove_tags = [dict(name=['meta','link'])]
+    remove_attributes=['lang']
+    keep_only_tags=[
+                      dict(name='h1', attrs={'class':'headline'})
+                     ,dict(attrs={'class':['author','post-time','body-copy']})
+                   ]
+
+    feeds = [(u'News', u'http://feeds.feedburner.com/TechCrunch/')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'            
+        return soup

recipes/tijolaco.recipe

@@ -0,0 +1,24 @@
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class Tijolaco(BasicNewsRecipe):
+    title                  = u'Tijolaco.com'
+    __author__             = u'Diniz Bortolotto'
+    description            = u'Posts do Blog Tijola\xe7o.com'
+    oldest_article         = 7
+    max_articles_per_feed  = 50
+    encoding               = 'utf8'
+    publisher              = u'Brizola Neto'
+    category               = 'politics, Brazil'
+    language               = 'pt_BR'
+    publication_type       = 'politics portal'
+    use_embedded_content   = False
+    no_stylesheets         = True
+    remove_javascript      = True
+
+    feeds                  = [(u'Blog Tijola\xe7o.com', u'http://feeds.feedburner.com/Tijolacoblog')]
+
+    reverse_article_order  = True
+
+    keep_only_tags         = [dict(name='div', attrs={'class':'post'})]
+
+    remove_tags            = [dict(name='span', attrs={'class':'com'})]

recipes/time_magazine.recipe

@@ -8,47 +8,33 @@ time.com
 
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
+from lxml import html
 
 class Time(BasicNewsRecipe):
     #recipe_disabled = ('This recipe has been disabled as TIME no longer'
     #        ' publish complete articles on the web.')
     title                 = u'Time'
-    __author__            = 'Kovid Goyal and Sujata Raman'
+    __author__            = 'Kovid Goyal'
     description           = 'Weekly magazine'
     encoding = 'utf-8'
     no_stylesheets        = True
     language = 'en'
     remove_javascript     = True
 
-    extra_css      = ''' h1 {font-family:georgia,serif;color:#000000;}
-                        .mainHd{font-family:georgia,serif;color:#000000;}
-                         h2 {font-family:Arial,Sans-serif;}
-                        .name{font-family:Arial,Sans-serif; font-size:x-small;font-weight:bold; }
-                        .date{font-family:Arial,Sans-serif; font-size:x-small ;color:#999999;}
-                        .byline{font-family:Arial,Sans-serif; font-size:x-small ;}
-                        .photoBkt{ font-size:x-small ;}
-                        .vertPhoto{font-size:x-small ;}
-                        .credits{font-family:Arial,Sans-serif; font-size:x-small ;color:gray;}
-                        .credit{font-family:Arial,Sans-serif; font-size:x-small ;color:gray;}
-                        .artTxt{font-family:georgia,serif;}
-                        #content{font-family:georgia,serif;}
-                        .caption{font-family:georgia,serif; font-size:x-small;color:#333333;}
-                        .credit{font-family:georgia,serif; font-size:x-small;color:#999999;}
-                        a:link{color:#CC0000;}
-                        .breadcrumb{font-family:Arial,Sans-serif;font-size:x-small;}
-                        '''
 
-
-    keep_only_tags = [ dict(name ="div",attrs = {"id" :["content"]}) ,
-                        dict(name ="div",attrs = {"class" :["artHd","artTxt","photoBkt","vertPhoto","image","copy"]}) ,]
-    remove_tags    = [           dict(name ="div",attrs =  {'class':['articleFooterNav','listsByTopic','articleTools2','relatedContent','sideContent','topBannerWrap','articlePagination','nextUp',"rtCol","pagination","enlarge","contentTools2",]}),
-                                  dict(name ="span",attrs =  {'class':['see']}),
-                                 dict(name ="div",attrs =  {'id':['header','articleSideBar',"articleTools","articleFooter","cmBotLt","quigoPackage"]}),
-                                  dict(name ="a",attrs =  {'class':['listLink']}),
-                                 dict(name ="ul",attrs =  {'id':['shareSocial','tabs']}),
-                                 dict(name ="li",attrs =  {'class':['back']}),
-                                 dict(name ="ul",attrs =  {'class':['navCount']}),
+    keep_only_tags = [
+            {
+                'class':['artHd', 'articleContent',
+                            'entry-title','entry-meta', 'entry-content', 'thumbnail']
+            },
         ]
+    remove_tags = [
+            {'class':['content-tools', 'quigo', 'see',
+                'first-tier-social-tools', 'navigation', 'enlarge lightbox']},
+            {'id':['share-tools']},
+            {'rel':'lightbox'},
+            ]
+
     recursions = 10
     match_regexps = [r'/[0-9,]+-(2|3|4|5|6|7|8|9)(,\d+){0,1}.html',r'http://www.time.com/time/specials/packages/article/.*']
 
@@ -56,10 +42,11 @@ class Time(BasicNewsRecipe):
         r'<meta .+/>'), lambda m:'')]
 
     def parse_index(self):
-        soup = self.index_to_soup('http://www.time.com/time/magazine')
-        img = soup.find('a', title="View Large Cover", href=True)
-        if img is not None:
-            cover_url = 'http://www.time.com'+img['href']
+        raw = self.index_to_soup('http://www.time.com/time/magazine', raw=True)
+        root = html.fromstring(raw)
+        img = root.xpath('//a[.="View Large Cover" and @href]')
+        if img:
+            cover_url = 'http://www.time.com' + img[0].get('href')
             try:
                 nsoup = self.index_to_soup(cover_url)
                 img = nsoup.find('img', src=re.compile('archive/covers'))
@@ -70,46 +57,48 @@ class Time(BasicNewsRecipe):
 
 
         feeds = []
-        parent = soup.find(id='tocGuts')
-        for seched in parent.findAll(attrs={'class':'toc_seched'}):
-            section = self.tag_to_string(seched).capitalize()
-            articles = list(self.find_articles(seched))
+        parent = root.xpath('//div[@class="content-main-aside"]')[0]
+        for sec in parent.xpath(
+                'descendant::section[contains(@class, "sec-mag-section")]'):
+            h3 = sec.xpath('./h3')
+            if h3:
+                section = html.tostring(h3[0], encoding=unicode,
+                        method='text').strip().capitalize()
+                self.log('Found section', section)
+                articles = list(self.find_articles(sec))
+                if articles:
                     feeds.append((section, articles))
 
         return feeds
 
-    def find_articles(self, seched):
-            for a in seched.findNextSiblings( attrs={'class':['toc_hed','rule2']}):
-              if a.name in "div":
-                  break
-              else:
+    def find_articles(self, sec):
+
+        for article in sec.xpath('./article'):
+            h2 = article.xpath('./*[@class="entry-title"]')
+            if not h2: continue
+            a = h2[0].xpath('./a[@href]')
+            if not a: continue
+            title = html.tostring(a[0], encoding=unicode,
+                        method='text').strip()
+            if not title: continue
+            url =  a[0].get('href')
+            if url.startswith('/'):
+                url = 'http://www.time.com'+url
+            desc = ''
+            p = article.xpath('./*[@class="entry-content"]')
+            if p:
+                desc = html.tostring(p[0], encoding=unicode,
+                        method='text')
+            self.log('\t', title, ':\n\t\t', desc)
             yield {
-                           'title' : self.tag_to_string(a),
-                           'url'   : 'http://www.time.com'+a['href'],
+                    'title' : title,
+                    'url'   : url,
                     'date'  : '',
-                           'description' : self.article_description(a)
+                    'description' : desc
                     }
 
-
-
-    def article_description(self, a):
-        ans = []
-        while True:
-            t = a.nextSibling
-            if t is None:
-                break
-            a = t
-            if getattr(t, 'name', False):
-                if t.get('class', '') == 'toc_parens' or t.name == 'br':
-                    continue
-                if t.name in ('div', 'a'):
-                    break
-                ans.append(self.tag_to_string(t))
-            else:
-                ans.append(unicode(t))
-        return u' '.join(ans).replace(u'\xa0', u'').strip()
-
     def postprocess_html(self,soup,first):
         for tag in soup.findAll(attrs ={'class':['artPag','pagination']}):
             tag.extract()
         return soup
+

recipes/vio_mundo.recipe

@@ -0,0 +1,30 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class VioMundo(BasicNewsRecipe):
+    title                  = 'Blog VioMundo'
+    __author__             = 'Diniz Bortolotto'
+    description            = 'Posts do Blog VioMundo'
+    publisher              = 'Luiz Carlos Azenha'
+    oldest_article         = 5
+    max_articles_per_feed  = 20
+    category               = 'news, politics, Brazil'
+    language               = 'pt_BR'
+    publication_type       = 'news and politics portal'
+    use_embedded_content   = False
+    no_stylesheets         = True
+    remove_javascript      = True
+
+    feeds                  = [(u'Blog VioMundo', u'http://www.viomundo.com.br/feed')]
+
+    reverse_article_order  = True
+
+    def print_version(self, url):
+        return url + '/print/'
+
+    remove_tags_after      = dict(id='BlogContent')
+
+    preprocess_regexps     = [
+                              (re.compile(r'\|\ <u>.*</p>'),
+                              lambda match: '</p>')
+                             ]

recipes/wired_uk.recipe

@@ -1,28 +1,29 @@
-
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2011, Starson17 <Starson17 at gmail.com>'
 '''
 www.wired.co.uk
 '''
 
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
+import re
 
 class Wired_UK(BasicNewsRecipe):
     title                 = 'Wired Magazine - UK edition'
-    __author__            = 'Darko Miletic'
+    __author__            = 'Starson17'
+    __version__           = 'v1.30'
+    __date__              = '15 July 2011'
     description           = 'Gaming news'
     publisher             = 'Conde Nast Digital'
     category              = 'news, games, IT, gadgets'
-    oldest_article        = 32
+    oldest_article        = 40
     max_articles_per_feed = 100
     no_stylesheets        = True
     encoding              = 'utf-8'
     use_embedded_content  = False
-    masthead_url          = 'http://www.wired.co.uk/_/media/wired-logo_UK.gif'
+    #masthead_url          = 'http://www.wired.co.uk/_/media/wired-logo_UK.gif'
     language              = 'en_GB'
-    extra_css             = ' body{font-family: Palatino,"Palatino Linotype","Times New Roman",Times,serif} img{margin-bottom: 0.8em } .img-descr{font-family: Tahoma,Arial,Helvetica,sans-serif; font-size: 0.6875em; display: block} '
-    index                 = 'http://www.wired.co.uk/wired-magazine.aspx'
+    index                 = 'http://www.wired.co.uk'
 
     conversion_options = {
                           'comment'   : description
@@ -31,26 +32,25 @@ class Wired_UK(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    keep_only_tags = [dict(name='div', attrs={'class':'article-box'})]
-    remove_tags = [
-                     dict(name=['object','embed','iframe','link'])
+    keep_only_tags = [dict(name='div', attrs={'class':['layoutColumn1']})]
+    remove_tags = [dict(name='div',attrs={'class':['articleSidebar1','commentAddBox linkit','commentCountBox commentCountBoxBig']})]
+    remove_tags_after = dict(name='div',attrs={'class':['mainCopy entry-content','mainCopy']})
+    '''
+    remove_attributes = ['height','width']
+                   ,dict(name=['object','embed','iframe','link'])
                    ,dict(attrs={'class':['opts','comment','stories']})
                    ]
-    remove_tags_after = dict(name='div',attrs={'class':'stories'})
-    remove_attributes = ['height','width']
-
-
+    '''
     def parse_index(self):
         totalfeeds = []
         soup   = self.index_to_soup(self.index)
-        maincontent = soup.find('div',attrs={'class':'main-content'})
+        recentcontent = soup.find('ul',attrs={'class':'linkList3'})
         mfeed = []
-        if maincontent:
-           st = maincontent.find(attrs={'class':'most-wired-box'})
-           if st:
-              for itt in st.findAll('a',href=True):
-               url   = 'http://www.wired.co.uk' + itt['href']
-               title = self.tag_to_string(itt)
+        if recentcontent:
+          for li in recentcontent.findAll('li'):
+            a = li.h2.a
+            url  = self.index + a['href'] + '?page=all'
+            title = self.tag_to_string(a)
             description = ''
             date  = strftime(self.timefmt)
             mfeed.append({
@@ -59,16 +59,91 @@ class Wired_UK(BasicNewsRecipe):
                  ,'url'        :url
                  ,'description':description
                 })
-        totalfeeds.append(('Articles', mfeed))
+        totalfeeds.append(('Wired UK Magazine Latest News', mfeed))
+        popmagcontent = soup.findAll('div',attrs={'class':'sidebarLinkList'})
+        magcontent = popmagcontent[1]
+        mfeed2 = []
+        if magcontent:
+          a = magcontent.h3.a
+          if a:
+            url   = self.index + a['href'] + '?page=all'
+            title = self.tag_to_string(a)
+            description = ''
+            date  = strftime(self.timefmt)
+            mfeed2.append({
+                  'title'      :title
+                 ,'date'       :date
+                 ,'url'        :url
+                 ,'description':description
+                })
+          for li in magcontent.findAll('li'):
+            a = li.a
+            url   = self.index + a['href'] + '?page=all'
+            title = self.tag_to_string(a)
+            description = ''
+            date  = strftime(self.timefmt)
+            mfeed2.append({
+                  'title'      :title
+                 ,'date'       :date
+                 ,'url'        :url
+                 ,'description':description
+                })
+          totalfeeds.append(('Wired UK Magazine Features', mfeed2))
+
+        magsoup = self.index_to_soup(self.index + '/magazine')
+        startcontent = magsoup.find('h3',attrs={'class':'magSubSectionTitle titleStart'}).parent
+        mfeed3 = []
+        if startcontent:
+          for li in startcontent.findAll('li'):
+            a = li.a
+            url   = self.index + a['href'] + '?page=all'
+            title = self.tag_to_string(a)
+            description = ''
+            date  = strftime(self.timefmt)
+            mfeed3.append({
+                  'title'      :title
+                 ,'date'       :date
+                 ,'url'        :url
+                 ,'description':description
+                })
+          totalfeeds.append(('Wired UK Magazine More', mfeed3))
+
+        playcontent = magsoup.find('h3',attrs={'class':'magSubSectionTitle titlePlay'}).parent
+        mfeed4 = []
+        if playcontent:
+          for li in playcontent.findAll('li'):
+            a = li.a
+            url   = self.index + a['href'] + '?page=all'
+            title = self.tag_to_string(a)
+            description = ''
+            date  = strftime(self.timefmt)
+            mfeed4.append({
+                  'title'      :title
+                 ,'date'       :date
+                 ,'url'        :url
+                 ,'description':description
+                })
+          totalfeeds.append(('Wired UK Magazine Play', mfeed4))
         return totalfeeds
 
     def get_cover_url(self):
-        cover_url = None
-        soup = self.index_to_soup(self.index)
-        cover_item = soup.find('span', attrs={'class':'cover'})
+        cover_url = ''
+        soup = self.index_to_soup(self.index + '/magazine/archive')
+        cover_item = soup.find('div', attrs={'class':'image linkme'})
         if cover_item:
            cover_url = cover_item.img['src']
         return cover_url
 
-    def print_version(self, url):
-        return url + '?page=all'
+    def preprocess_html(self, soup):
+        for tag in soup.findAll(name='p'):
+            if tag.find(name='span', text=re.compile(r'This article was taken from.*', re.DOTALL|re.IGNORECASE)):
+                tag.extract()
+        return soup
+
+    extra_css = '''
+                    h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;}
+                    h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;}
+                    p{font-family:Arial,Helvetica,sans-serif;font-size:small;}
+                    body{font-family:Helvetica,Arial,sans-serif;font-size:small;}
+		'''
+

recipes/zeitde.recipe

@@ -15,6 +15,7 @@ class ZeitDe(BasicNewsRecipe):
     encoding = 'UTF-8'
 
     __author__ = 'Martin Pitt, Sujata Raman, Ingo Paschke and Marc Toensing'
+    no_stylesheets = True
 
     max_articles_per_feed = 40
 

recipes/zeitde_sub.recipe

@@ -2,18 +2,21 @@
 # -*- coding: utf-8 mode: python -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2010-2011, Steffen Siebert <calibre at steffensiebert.de>'
+__copyright__ = '2010, Steffen Siebert <calibre at steffensiebert.de>'
 __docformat__ = 'restructuredtext de'
-__version__   = '1.2'
+__version__   = '1.5'
 
 """
 Die Zeit EPUB
 """
 
-import os, urllib2, zipfile, re
+import os, zipfile, re, cStringIO
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre import walk
+from urlparse import urlparse
+from contextlib import closing
+from calibre.utils.magick.draw import save_cover_data_to
 
 class ZeitEPUBAbo(BasicNewsRecipe):
 
@@ -22,49 +25,112 @@ class ZeitEPUBAbo(BasicNewsRecipe):
     language = 'de'
     lang = 'de-DE'
 
-    __author__ = 'Steffen Siebert and Tobias Isenberg'
+    __author__ = 'Steffen Siebert, revised by Tobias Isenberg (with some code by Kovid Goyal)'
     needs_subscription = True
 
     conversion_options = {
         'no_default_epub_cover' : True,
         # fixing the wrong left margin
         'mobi_ignore_margins' : True,
+        'keep_ligatures' : True,
     }
 
     preprocess_regexps    = [
-        # filtering for correct dashes
-        (re.compile(r' - '), lambda match: ' – '), # regular "Gedankenstrich"
-        (re.compile(r' -,'), lambda match: ' –,'), # "Gedankenstrich" before a comma
-        (re.compile(r'(?<=\d)-(?=\d)'), lambda match: '–'), # number-number
+        # filtering for correct dashes ("Gedankenstrich" and "bis")
+        (re.compile(u' (-|\u2212)(?=[ ,])'), lambda match: u' \u2013'),
+        (re.compile(r'(?<=\d)-(?=\d)'), lambda match: u'\u2013'), # number-number
+        (re.compile(u'(?<=\d,)-(?= ?\u20AC)'), lambda match: u'\u2013'), # ,- Euro
+        # fix the number dash number dash for the title image that was broken by the previous line
+        (re.compile(u'(?<=\d\d\d\d)\u2013(?=\d?\d\.png)'), lambda match: '-'),
+        # filtering for certain dash cases
+        (re.compile(r'Bild - Zeitung'), lambda match: 'Bild-Zeitung'), # the obvious
+        (re.compile(r'EMail'), lambda match: 'E-Mail'), # the obvious
+        (re.compile(r'SBahn'), lambda match: 'S-Bahn'), # the obvious
+        (re.compile(r'UBoot'), lambda match: 'U-Boot'), # the obvious
+        (re.compile(r'T Shirt'), lambda match: 'T-Shirt'), # the obvious
+        (re.compile(r'TShirt'), lambda match: 'T-Shirt'), # the obvious
+        # the next two lines not only fix errors but also create new ones. this is due to additional errors in
+        # the typesetting such as missing commas or wrongly placed dashes. but more is fixed than broken.
+        (re.compile(r'(?<!und|der|\w\w,) -(?=\w)'), lambda match: '-'), # space too much before a connecting dash
+        (re.compile(r'(?<=\w)- (?!und\b|oder\b|wie\b|aber\b|auch\b|sondern\b|bis\b|&amp;|&\s|bzw\.|auf\b|eher\b)'), lambda match: '-'), # space too much after a connecting dash
+        # filtering for missing spaces before the month in long dates
+        (re.compile(u'(?<=\d)\.(?=(Januar|Februar|M\u00E4rz|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember))'), lambda match: '. '),
+        # filtering for other missing spaces
+        (re.compile(r'Stuttgart21'), lambda match: 'Stuttgart 21'), # the obvious
+        (re.compile(u'(?<=\d)(?=\u20AC)'), lambda match: u'\u2013'), # Zahl[no space]Euro
+        (re.compile(r':(?=[^\d\s</])'), lambda match: ': '), # missing space after colon
+        (re.compile(u'\u00AB(?=[^\-\.:;,\?!<\)\s])'), lambda match: u'\u00AB '), # missing space after closing quotation
+        (re.compile(u'(?<=[^\s\(>])\u00BB'), lambda match: u' \u00BB'), # missing space before opening quotation
+        (re.compile(r'(?<=[a-z])(?=(I|II|III|IV|V|VI|VII|VIII|IX|X|XI|XII|XIII|XIV|XV|XVI|XVII|XVIII|XIX|XX)\.)'), lambda match: ' '), # missing space before Roman numeral
+        (re.compile(r'(?<=(I|V|X)\.)(?=[\w])'), lambda match: ' '), # missing space after Roman numeral
+        (re.compile(r'(?<=(II|IV|VI|IX|XI|XV|XX)\.)(?=[\w])'), lambda match: ' '), # missing space after Roman numeral
+        (re.compile(r'(?<=(III|VII|XII|XIV|XVI|XIX)\.)(?=[\w])'), lambda match: ' '), # missing space after Roman numeral
+        (re.compile(r'(?<=(VIII|XIII|XVII)\.)(?=[\w])'), lambda match: ' '), # missing space after Roman numeral
+        (re.compile(r'(?<=(XVIII)\.)(?=[\w])'), lambda match: ' '), # missing space after Roman numeral
+        (re.compile(r'(?<=[A-Za-zÄÖÜäöü]),(?=[A-Za-zÄÖÜäöü])'), lambda match: ', '), # missing space after comma
+        (re.compile(r'(?<=[a-zäöü])\.(?=[A-ZÄÖÜ][A-Za-zÄÖÜäöü])'), lambda match: '. '), # missing space after full-stop
+        (re.compile(r'(?<=[uU]\.) (?=a\.)'), lambda match: u'\u2008'), # fix abbreviation that was potentially broken previously
+        (re.compile(r'(?<=[iI]\.) (?=A\.)'), lambda match: u'\u2008'), # fix abbreviation that was potentially broken previously
+        (re.compile(r'(?<=[zZ]\.) (?=B\.)'), lambda match: u'\u2008'), # fix abbreviation that was potentially broken previously
+        (re.compile(r'(?<=\w\.) (?=[A-Z][a-z]*@)'), lambda match: ''), # fix e-mail address that was potentially broken previously
+        (re.compile(r'(?<=\d)[Pp]rozent'), lambda match: ' Prozent'),
+        (re.compile(r'\.\.\.\.+'), lambda match: '...'), # too many dots (....)
+        (re.compile(r'(?<=[^\s])\.\.\.'), lambda match: ' ...'), # spaces before ...
+        (re.compile(r'\.\.\.(?=[^\s])'), lambda match: '... '), # spaces after ...
+        (re.compile(r'(?<=[\[\(]) \.\.\. (?=[\]\)])'), lambda match: '...'), # fix special cases of ... in brackets
+        (re.compile(u'(?<=[\u00BB\u203A]) \.\.\.'), lambda match: '...'), # fix special cases of ... after a quotation mark
+        (re.compile(u'\.\.\. (?=[\u00AB\u2039,])'), lambda match: '...'), # fix special cases of ... before a quotation mark or comma
+        # fix missing spaces between numbers and any sort of units, possibly with dot
+        (re.compile(r'(?<=\d)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'), lambda match: ' '),
+        (re.compile(r'(?<=\d\.)(?=(Femto|Piko|Nano|Mikro|Milli|Zenti|Dezi|Hekto|Kilo|Mega|Giga|Tera|Peta|Tausend|Trilli|Kubik|Quadrat|Meter|Uhr|Jahr|Schuljahr|Seite))'), lambda match: ' '),
+        # fix wrong spaces
+        (re.compile(r'(?<=<p class="absatz">[A-ZÄÖÜ]) (?=[a-zäöü\-])'), lambda match: ''), # at beginning of paragraphs
+        (re.compile(u' \u00AB'), lambda match: u'\u00AB '), # before closing quotation
+        (re.compile(u'\u00BB '), lambda match: u' \u00BB'), # after opening quotation
+        # filtering for spaces in large numbers for better readability
+        (re.compile(r'(?<=\d\d)(?=\d\d\d[ ,\.;\)<\?!-])'), lambda match: u'\u2008'), # end of the number with some character following
+        (re.compile(r'(?<=\d\d)(?=\d\d\d. )'), lambda match: u'\u2008'), # end of the number with full-stop following, then space is necessary (avoid file names)
+        (re.compile(u'(?<=\d)(?=\d\d\d\u2008)'), lambda match: u'\u2008'), # next level
+        (re.compile(u'(?<=\d)(?=\d\d\d\u2008)'), lambda match: u'\u2008'), # next level
+        (re.compile(u'(?<=\d)(?=\d\d\d\u2008)'), lambda match: u'\u2008'), # next level
+        (re.compile(u'(?<=\d)(?=\d\d\d\u2008)'), lambda match: u'\u2008'), # next level
         # filtering for unicode characters that are missing on the Kindle,
         # try to replace them with meaningful work-arounds
-        (re.compile(u'\u2080'), lambda match: '<span style="font-size: 50%;">0</span>'), # subscript-0
-        (re.compile(u'\u2081'), lambda match: '<span style="font-size: 50%;">1</span>'), # subscript-1
-        (re.compile(u'\u2082'), lambda match: '<span style="font-size: 50%;">2</span>'), # subscript-2
-        (re.compile(u'\u2083'), lambda match: '<span style="font-size: 50%;">3</span>'), # subscript-3
-        (re.compile(u'\u2084'), lambda match: '<span style="font-size: 50%;">4</span>'), # subscript-4
-        (re.compile(u'\u2085'), lambda match: '<span style="font-size: 50%;">5</span>'), # subscript-5
-        (re.compile(u'\u2086'), lambda match: '<span style="font-size: 50%;">6</span>'), # subscript-6
-        (re.compile(u'\u2087'), lambda match: '<span style="font-size: 50%;">7</span>'), # subscript-7
-        (re.compile(u'\u2088'), lambda match: '<span style="font-size: 50%;">8</span>'), # subscript-8
-        (re.compile(u'\u2089'), lambda match: '<span style="font-size: 50%;">9</span>'), # subscript-9
+        (re.compile(u'\u2080'), lambda match: '<span style="font-size: 40%;">0</span>'), # subscript-0
+        (re.compile(u'\u2081'), lambda match: '<span style="font-size: 40%;">1</span>'), # subscript-1
+        (re.compile(u'\u2082'), lambda match: '<span style="font-size: 40%;">2</span>'), # subscript-2
+        (re.compile(u'\u2083'), lambda match: '<span style="font-size: 40%;">3</span>'), # subscript-3
+        (re.compile(u'\u2084'), lambda match: '<span style="font-size: 40%;">4</span>'), # subscript-4
+        (re.compile(u'\u2085'), lambda match: '<span style="font-size: 40%;">5</span>'), # subscript-5
+        (re.compile(u'\u2086'), lambda match: '<span style="font-size: 40%;">6</span>'), # subscript-6
+        (re.compile(u'\u2087'), lambda match: '<span style="font-size: 40%;">7</span>'), # subscript-7
+        (re.compile(u'\u2088'), lambda match: '<span style="font-size: 40%;">8</span>'), # subscript-8
+        (re.compile(u'\u2089'), lambda match: '<span style="font-size: 40%;">9</span>'), # subscript-9
+        # always chance CO2
+        (re.compile(r'CO2'), lambda match: 'CO<span style="font-size: 40%;">2</span>'), # CO2
+        # remove *** paragraphs
+        (re.compile(r'<p class="absatz">\*\*\*</p>'), lambda match: ''),
+        # better layout for the top line of each article
+        (re.compile(u'(?<=DIE ZEIT N\u00B0 \d /) (?=\d\d)'), lambda match: ' 20'), # proper year in edition number
+        (re.compile(u'(?<=DIE ZEIT N\u00B0 \d\d /) (?=\d\d)'), lambda match: ' 20'), # proper year in edition number
+        (re.compile(u'(?<=>)(?=DIE ZEIT N\u00B0 \d\d / 20\d\d)'), lambda match: u' \u2014 '), # m-dash between category and DIE ZEIT
     ]
 
     def build_index(self):
-        domain = "http://premium.zeit.de"
-        url = domain + "/abovorteile/cgi-bin/_er_member/p4z.fpl?ER_Do=getUserData&ER_NextTemplate=login_ok"
-
+        domain = "https://premium.zeit.de"
+        url = domain + "/abo/zeit_digital"
         browser = self.get_browser()
-        browser.add_password("http://premium.zeit.de", self.username, self.password)
 
-        try:
-            browser.open(url)
-        except urllib2.HTTPError:
-            self.report_progress(0,_("Can't login to download issue"))
-            raise ValueError('Failed to login, check your username and password')
-
-        response = browser.follow_link(text="DIE ZEIT als E-Paper")
-        response = browser.follow_link(url_regex=re.compile('^http://contentserver.hgv-online.de/nodrm/fulfillment\\?distributor=zeit-online&orderid=zeit_online.*'))
+        # new login process
+        response = browser.open(url)
+        browser.select_form(nr=2)
+        browser.form['name']=self.username
+        browser.form['pass']=self.password
+        browser.submit()
+        # now find the correct file, we will still use the ePub file
+        epublink = browser.find_link(text_regex=re.compile('.*Ausgabe als Datei im ePub-Format.*'))
+        response = browser.follow_link(epublink)
+        self.report_progress(1,_('next step'))
 
         tmp = PersistentTemporaryFile(suffix='.epub')
         self.report_progress(0,_('downloading epub'))
@@ -104,9 +170,45 @@ class ZeitEPUBAbo(BasicNewsRecipe):
 
     # getting url of the cover
     def get_cover_url(self):
+        self.log.warning('Downloading cover')
+        try:
+            self.log.warning('Trying PDF-based cover')
+            domain = "https://premium.zeit.de"
+            url = domain + "/abo/zeit_digital"
+            browser = self.get_browser()
+
+            # new login process
+            browser.open(url)
+            browser.select_form(nr=2)
+            browser.form['name']=self.username
+            browser.form['pass']=self.password
+            browser.submit()
+            # actual cover search
+            pdflink = browser.find_link(url_regex=re.compile('system/files/epaper/DZ/pdf/DZ_ePaper*'))
+            cover_url = urlparse(pdflink.base_url)[0]+'://'+urlparse(pdflink.base_url)[1]+''+(urlparse(pdflink.url)[2]).replace('ePaper_','').replace('.pdf','_001.pdf')
+            self.log.warning('PDF link found:')
+            self.log.warning(cover_url)
+            # download the cover (has to be here due to new login process)
+            with closing(browser.open(cover_url)) as r:
+                cdata = r.read()
+            from calibre.ebooks.metadata.pdf import get_metadata
+            stream = cStringIO.StringIO(cdata)
+            cdata = None
+            mi = get_metadata(stream)
+            if mi.cover_data and mi.cover_data[1]:
+                cdata = mi.cover_data[1]
+
+            cpath = os.path.join(self.output_dir, 'cover.jpg')
+            save_cover_data_to(cdata, cpath)
+            cover_url = cpath
+
+        except:
+            self.log.warning('Trying low-res cover')
             try:
                 inhalt = self.index_to_soup('http://www.zeit.de/inhalt')
                 cover_url = inhalt.find('div', attrs={'class':'singlearchive clearfix'}).img['src'].replace('icon_','')
             except:
+                self.log.warning('Using static old low-res cover')
                 cover_url = 'http://images.zeit.de/bilder/titelseiten_zeit/1946/001_001.jpg'
         return cover_url
+

resources/content_server/browse/browse.html

@@ -11,6 +11,7 @@
     <link rel="stylesheet" type="text/css" href="{prefix}/static/browse/browse.css" />
     <link type="text/css" href="{prefix}/static/jquery_ui/css/humanity-custom/jquery-ui-1.8.5.custom.css" rel="stylesheet" />
     <link rel="stylesheet" type="text/css" href="{prefix}/static/jquery.multiselect.css" />
+    <link rel="apple-touch-icon" href="/static/calibre.png" />
 
     <script type="text/javascript" src="{prefix}/static/jquery.js"></script>
     <script type="text/javascript" src="{prefix}/static/jquery.corner.js"></script>

resources/default_tweaks.py

@@ -366,3 +366,10 @@ server_listen_on = '0.0.0.0'
 # on at your own risk!
 unified_title_toolbar_on_osx = False
 
+#: Save original file when converting from same format to same format
+# When calibre does a conversion from the same format to the same format, for
+# example, from EPUB to EPUB, the original file is saved, so that in case the
+# conversion is poor, you can tweak the settings and run it again. By setting
+# this to False you can prevent calibre from saving the original file.
+save_original_format = True
+

resources/templates/fb2.xsl

@@ -379,7 +379,8 @@
     <!-- image -->
     <xsl:template match="fb:image">
         <div align="center">
-            <img border="1">
+            <xsl:element name="img">
+                <xsl:attribute name="border">1</xsl:attribute>
                 <xsl:choose>
                     <xsl:when test="starts-with(@xlink:href,'#')">
                         <xsl:attribute name="src"><xsl:value-of select="substring-after(@xlink:href,'#')"/></xsl:attribute>
@@ -388,7 +389,10 @@
                         <xsl:attribute name="src"><xsl:value-of select="@xlink:href"/></xsl:attribute>
                     </xsl:otherwise>
                 </xsl:choose>
-            </img>
+                <xsl:if test="@title">
+                    <xsl:attribute name="title"><xsl:value-of select="@title"/></xsl:attribute>
+                </xsl:if>
+            </xsl:element>
         </div>
     </xsl:template>
 </xsl:stylesheet>

session.vim

@@ -1,5 +1,5 @@
 " Project wide builtins
-let g:pyflakes_builtins += ["dynamic_property", "__", "P", "I", "lopen", "icu_lower", "icu_upper", "icu_title", "ngettext"]
+let g:pyflakes_builtins = ["_", "dynamic_property", "__", "P", "I", "lopen", "icu_lower", "icu_upper", "icu_title", "ngettext"]
 
 python << EOFPY
 import os
@@ -15,7 +15,7 @@ vipy.session.initialize(project_name='calibre', src_dir=src_dir,
             project_dir=project_dir, base_dir=base_dir)
 
 def recipe_title_callback(raw):
-    return eval(raw.decode('utf-8'))
+    return eval(raw.decode('utf-8')).replace(' ', '_')
 
 vipy.session.add_content_browser('.r', ',r', 'Recipe',
     vipy.session.glob_based_iterator(os.path.join(project_dir, 'recipes', '*.recipe')),

setup/check.py

@@ -25,18 +25,11 @@ class Message:
         return '%s:%s: %s'%(self.filename, self.lineno, self.msg)
 
 def check_for_python_errors(code_string, filename):
-    # Since compiler.parse does not reliably report syntax errors, use the
-    # built in compiler first to detect those.
+    import _ast
+    # First, compile into an AST and handle syntax errors.
     try:
-        try:
-            compile(code_string, filename, "exec")
-        except MemoryError:
-            # Python 2.4 will raise MemoryError if the source can't be
-            # decoded.
-            if sys.version_info[:2] == (2, 4):
-                raise SyntaxError(None)
-            raise
-    except (SyntaxError, IndentationError), value:
+        tree = compile(code_string, filename, "exec", _ast.PyCF_ONLY_AST)
+    except (SyntaxError, IndentationError) as value:
         msg = value.args[0]
 
         (lineno, offset, text) = value.lineno, value.offset, value.text
@@ -47,13 +40,11 @@ def check_for_python_errors(code_string, filename):
             # bogus message that claims the encoding the file declared was
             # unknown.
             msg = "%s: problem decoding source" % filename
+
         return [Message(filename, lineno, msg)]
     else:
-        # Okay, it's syntactically valid.  Now parse it into an ast and check
-        # it.
-        import compiler
         checker = __import__('pyflakes.checker').checker
-        tree = compiler.parse(code_string)
+        # Okay, it's syntactically valid.  Now check it.
         w = checker.Checker(tree, filename)
         w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
         return [Message(x.filename, x.lineno, x.message%x.message_args) for x in

setup/translations.py

@@ -8,11 +8,18 @@ __docformat__ = 'restructuredtext en'
 
 import os, tempfile, shutil, subprocess, glob, re, time, textwrap
 from distutils import sysconfig
+from functools import partial
 
 from setup import Command, __appname__, __version__
-from setup.build_environment import pyqt
 
-class POT(Command):
+def qt_sources():
+    qtdir = glob.glob('/usr/src/qt-*')[-1]
+    j = partial(os.path.join, qtdir)
+    return list(map(j, [
+            'src/gui/widgets/qdialogbuttonbox.cpp',
+    ]))
+
+class POT(Command): # {{{
 
     description = 'Update the .pot translation template'
     PATH = os.path.join(Command.SRC, __appname__, 'translations')
@@ -82,6 +89,8 @@ class POT(Command):
                 time=time.strftime('%Y-%m-%d %H:%M+%Z'))
 
         files = self.source_files()
+        qt_inputs = qt_sources()
+
         with tempfile.NamedTemporaryFile() as fl:
             fl.write('\n'.join(files))
             fl.flush()
@@ -91,8 +100,14 @@ class POT(Command):
             subprocess.check_call(['xgettext', '-f', fl.name,
                 '--default-domain=calibre', '-o', out.name, '-L', 'Python',
                 '--from-code=UTF-8', '--sort-by-file', '--omit-header',
-                '--no-wrap', '-k__',
+                '--no-wrap', '-k__', '--add-comments=NOTE:',
                 ])
+            subprocess.check_call(['xgettext', '-j',
+                '--default-domain=calibre', '-o', out.name,
+                '--from-code=UTF-8', '--sort-by-file', '--omit-header',
+                '--no-wrap', '-kQT_TRANSLATE_NOOP:2',
+                ] + qt_inputs)
+
             with open(out.name, 'rb') as f:
                 src = f.read()
             os.remove(out.name)
@@ -102,10 +117,12 @@ class POT(Command):
             with open(pot, 'wb') as f:
                 f.write(src)
             self.info('Translations template:', os.path.abspath(pot))
+
+
         return pot
+# }}}
 
-
-class Translations(POT):
+class Translations(POT): # {{{
     description='''Compile the translations'''
     DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
             'locales')
@@ -117,7 +134,6 @@ class Translations(POT):
         locale = os.path.splitext(os.path.basename(po_file))[0]
         return locale, os.path.join(self.DEST, locale, 'messages.mo')
 
-
     def run(self, opts):
         for f in self.po_files():
             locale, dest = self.mo_file(f)
@@ -126,7 +142,7 @@ class Translations(POT):
                 os.makedirs(base)
             self.info('\tCompiling translations for', locale)
             subprocess.check_call(['msgfmt', '-o', dest, f])
-            if locale in ('en_GB', 'nds', 'te', 'yi'):
+            if locale in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi'):
                 continue
             pycountry = self.j(sysconfig.get_python_lib(), 'pycountry',
                     'locales', locale, 'LC_MESSAGES')
@@ -140,17 +156,6 @@ class Translations(POT):
                 self.warn('No ISO 639 translations for locale:', locale,
                 '\nDo you have pycountry installed?')
 
-        base = os.path.join(pyqt.qt_data_dir, 'translations')
-        qt_translations = glob.glob(os.path.join(base, 'qt_*.qm'))
-        if not qt_translations:
-            raise Exception('Could not find qt translations')
-        for f in qt_translations:
-            locale = self.s(self.b(f))[0][3:]
-            dest = self.j(self.DEST, locale, 'LC_MESSAGES', 'qt.qm')
-            if self.e(self.d(dest)) and self.newer(dest, f):
-                self.info('\tCopying Qt translation for locale:', locale)
-                shutil.copy2(f, dest)
-
         self.write_stats()
         self.freeze_locales()
 
@@ -201,7 +206,7 @@ class Translations(POT):
             for x in (i, j, d):
                 if os.path.exists(x):
                     os.remove(x)
-
+# }}}
 
 class GetTranslations(Translations):
 

src/calibre/__init__.py

@@ -341,7 +341,7 @@ def random_user_agent():
 def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None):
     '''
     Create a mechanize browser for web scraping. The browser handles cookies,
-    refresh requests and ignores robots.txt. Also uses proxy if avaialable.
+    refresh requests and ignores robots.txt. Also uses proxy if available.
 
     :param honor_time: If True honors pause time in refresh requests
     :param max_time: Maximum time in seconds to wait during a refresh request
@@ -474,7 +474,7 @@ def strftime(fmt, t=None):
 def my_unichr(num):
     try:
         return unichr(num)
-    except ValueError:
+    except (ValueError, OverflowError):
         return u'?'
 
 def entity_to_unicode(match, exceptions=[], encoding='cp1252',

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 9)
+numeric_version = (0, 8, 10)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/db/backend.py

@@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 # Imports {{{
-import os, shutil, uuid, json
+import os, shutil, uuid, json, glob, time, tempfile
 from functools import partial
 
 import apsw
@@ -25,7 +25,7 @@ from calibre.utils.config import to_json, from_json, prefs, tweaks
 from calibre.utils.date import utcfromtimestamp, parse_date
 from calibre.utils.filenames import is_case_sensitive
 from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
-        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable)
+        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, CompositeTable)
 # }}}
 
 '''
@@ -37,6 +37,8 @@ Differences in semantics from pysqlite:
 
 '''
 
+SPOOL_SIZE = 30*1024*1024
+
 class DynamicFilter(object): # {{{
 
     'No longer used, present for legacy compatibility'
@@ -478,7 +480,6 @@ class DB(object):
                 remove.append(data)
                 continue
 
-            self.custom_column_label_map[data['label']] = data['num']
             self.custom_column_num_map[data['num']] = \
                 self.custom_column_label_map[data['label']] = data
 
@@ -613,10 +614,31 @@ class DB(object):
 
         tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
 
-        for label, data in self.custom_column_label_map.iteritems():
-            label = '#' + label
+        self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3,
+             'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
+             'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12,
+             'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17,
+             'au_map':18, 'last_modified':19, 'identifiers':20}
+
+        for k,v in self.FIELD_MAP.iteritems():
+            self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
+
+        base = max(self.FIELD_MAP.itervalues())
+
+        for label_, data in self.custom_column_label_map.iteritems():
+            label = self.field_metadata.custom_field_prefix + label_
             metadata = self.field_metadata[label].copy()
             link_table = self.custom_table_names(data['num'])[1]
+            self.FIELD_MAP[data['num']] = base = base+1
+            self.field_metadata.set_field_record_index(label_, base,
+                    prefer_custom=True)
+            if data['datatype'] == 'series':
+                # account for the series index column. Field_metadata knows that
+                # the series index is one larger than the series. If you change
+                # it here, be sure to change it there as well.
+                self.FIELD_MAP[str(data['num'])+'_index'] = base = base+1
+                self.field_metadata.set_field_record_index(label_+'_index', base,
+                            prefer_custom=True)
 
             if data['normalized']:
                 if metadata['is_multiple']:
@@ -632,8 +654,17 @@ class DB(object):
                         metadata['column'] = 'extra'
                         metadata['table'] = link_table
                         tables[label] = OneToOneTable(label, metadata)
+            else:
+                if data['datatype'] == 'composite':
+                    tables[label] = CompositeTable(label, metadata)
                 else:
                     tables[label] = OneToOneTable(label, metadata)
+
+        self.FIELD_MAP['ondevice'] = base = base+1
+        self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False)
+        self.FIELD_MAP['marked'] = base = base+1
+        self.field_metadata.set_field_record_index('marked', base, prefer_custom=False)
+
     # }}}
 
     @property
@@ -732,5 +763,57 @@ class DB(object):
                     pprint.pprint(table.metadata)
                     raise
 
+    def format_abspath(self, book_id, fmt, fname, path):
+        path = os.path.join(self.library_path, path)
+        fmt = ('.' + fmt.lower()) if fmt else ''
+        fmt_path = os.path.join(path, fname+fmt)
+        if os.path.exists(fmt_path):
+            return fmt_path
+        try:
+            candidates = glob.glob(os.path.join(path, '*'+fmt))
+        except: # If path contains strange characters this throws an exc
+            candidates = []
+        if fmt and candidates and os.path.exists(candidates[0]):
+            shutil.copyfile(candidates[0], fmt_path)
+            return fmt_path
+
+    def format_metadata(self, book_id, fmt, fname, path):
+        path = self.format_abspath(book_id, fmt, fname, path)
+        ans = {}
+        if path is not None:
+            stat = os.stat(path)
+            ans['size'] = stat.st_size
+            ans['mtime'] = utcfromtimestamp(stat.st_mtime)
+        return ans
+
+    def cover(self, path, as_file=False, as_image=False,
+            as_path=False):
+        path = os.path.join(self.library_path, path, 'cover.jpg')
+        ret = None
+        if os.access(path, os.R_OK):
+            try:
+                f = lopen(path, 'rb')
+            except (IOError, OSError):
+                time.sleep(0.2)
+                f = lopen(path, 'rb')
+            with f:
+                if as_path:
+                    pt = PersistentTemporaryFile('_dbcover.jpg')
+                    with pt:
+                        shutil.copyfileobj(f, pt)
+                    return pt.name
+                if as_file:
+                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
+                    shutil.copyfileobj(f, ret)
+                    ret.seek(0)
+                else:
+                    ret = f.read()
+                    if as_image:
+                        from PyQt4.Qt import QImage
+                        i = QImage()
+                        i.loadFromData(ret)
+                        ret = i
+        return ret
+
    # }}}
 

src/calibre/db/cache.py

@@ -7,5 +7,380 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
+from collections import defaultdict
+from functools import wraps, partial
+
+from calibre.db.locking import create_locks, RecordLock
+from calibre.db.fields import create_field
+from calibre.ebooks.book.base import Metadata
+from calibre.utils.date import now
+
+def api(f):
+    f.is_cache_api = True
+    return f
+
+def read_api(f):
+    f = api(f)
+    f.is_read_api = True
+    return f
+
+def write_api(f):
+    f = api(f)
+    f.is_read_api = False
+    return f
+
+def wrap_simple(lock, func):
+    @wraps(func)
+    def ans(*args, **kwargs):
+        with lock:
+            return func(*args, **kwargs)
+    return ans
 
 
+class Cache(object):
+
+    def __init__(self, backend):
+        self.backend = backend
+        self.fields = {}
+        self.composites = set()
+        self.read_lock, self.write_lock = create_locks()
+        self.record_lock = RecordLock(self.read_lock)
+        self.format_metadata_cache = defaultdict(dict)
+
+        # Implement locking for all simple read/write API methods
+        # An unlocked version of the method is stored with the name starting
+        # with a leading underscore. Use the unlocked versions when the lock
+        # has already been acquired.
+        for name in dir(self):
+            func = getattr(self, name)
+            ira = getattr(func, 'is_read_api', None)
+            if ira is not None:
+                # Save original function
+                setattr(self, '_'+name, func)
+                # Wrap it in a lock
+                lock = self.read_lock if ira else self.write_lock
+                setattr(self, name, wrap_simple(lock, func))
+
+    @property
+    def field_metadata(self):
+        return self.backend.field_metadata
+
+    def _format_abspath(self, book_id, fmt):
+        '''
+        Return absolute path to the ebook file of format `format`
+
+        WARNING: This method will return a dummy path for a network backend DB,
+        so do not rely on it, use format(..., as_path=True) instead.
+
+        Currently used only in calibredb list, the viewer and the catalogs (via
+        get_data_as_dict()).
+
+        Apart from the viewer, I don't believe any of the others do any file
+        I/O with the results of this call.
+        '''
+        try:
+            name = self.fields['formats'].format_fname(book_id, fmt)
+            path = self._field_for('path', book_id).replace('/', os.sep)
+        except:
+            return None
+        if name and path:
+            return self.backend.format_abspath(book_id, fmt, name, path)
+
+    def _get_metadata(self, book_id, get_user_categories=True): # {{{
+        mi = Metadata(None)
+        author_ids = self._field_ids_for('authors', book_id)
+        aut_list = [self._author_data(i) for i in author_ids]
+        aum = []
+        aus = {}
+        aul = {}
+        for rec in aut_list:
+            aut = rec['name']
+            aum.append(aut)
+            aus[aut] = rec['sort']
+            aul[aut] = rec['link']
+        mi.title       = self._field_for('title', book_id,
+                default_value=_('Unknown'))
+        mi.authors     = aum
+        mi.author_sort = self._field_for('author_sort', book_id,
+                default_value=_('Unknown'))
+        mi.author_sort_map = aus
+        mi.author_link_map = aul
+        mi.comments    = self._field_for('comments', book_id)
+        mi.publisher   = self._field_for('publisher', book_id)
+        n = now()
+        mi.timestamp   = self._field_for('timestamp', book_id, default_value=n)
+        mi.pubdate     = self._field_for('pubdate', book_id, default_value=n)
+        mi.uuid        = self._field_for('uuid', book_id,
+                default_value='dummy')
+        mi.title_sort  = self._field_for('sort', book_id,
+                default_value=_('Unknown'))
+        mi.book_size   = self._field_for('size', book_id, default_value=0)
+        mi.ondevice_col = self._field_for('ondevice', book_id, default_value='')
+        mi.last_modified = self._field_for('last_modified', book_id,
+                default_value=n)
+        formats = self._field_for('formats', book_id)
+        mi.format_metadata = {}
+        if not formats:
+            formats = None
+        else:
+            for f in formats:
+                mi.format_metadata[f] = self._format_metadata(book_id, f)
+            formats = ','.join(formats)
+        mi.formats = formats
+        mi.has_cover = _('Yes') if self._field_for('cover', book_id,
+                default_value=False) else ''
+        mi.tags = list(self._field_for('tags', book_id, default_value=()))
+        mi.series = self._field_for('series', book_id)
+        if mi.series:
+            mi.series_index = self._field_for('series_index', book_id,
+                    default_value=1.0)
+        mi.rating = self._field_for('rating', book_id)
+        mi.set_identifiers(self._field_for('identifiers', book_id,
+            default_value={}))
+        mi.application_id = book_id
+        mi.id = book_id
+        composites = {}
+        for key, meta in self.field_metadata.custom_iteritems():
+            mi.set_user_metadata(key, meta)
+            if meta['datatype'] == 'composite':
+                composites.append(key)
+            else:
+                mi.set(key, val=self._field_for(meta['label'], book_id),
+                        extra=self._field_for(meta['label']+'_index', book_id))
+        for c in composites:
+            mi.set(key, val=self._composite_for(key, book_id, mi))
+
+        user_cat_vals = {}
+        if get_user_categories:
+            user_cats = self.prefs['user_categories']
+            for ucat in user_cats:
+                res = []
+                for name,cat,ign in user_cats[ucat]:
+                    v = mi.get(cat, None)
+                    if isinstance(v, list):
+                        if name in v:
+                            res.append([name,cat])
+                    elif name == v:
+                        res.append([name,cat])
+                user_cat_vals[ucat] = res
+        mi.user_categories = user_cat_vals
+
+        return mi
+    # }}}
+
+    # Cache Layer API {{{
+
+    @api
+    def init(self):
+        '''
+        Initialize this cache with data from the backend.
+        '''
+        with self.write_lock:
+            self.backend.read_tables()
+
+            for field, table in self.backend.tables.iteritems():
+                self.fields[field] = create_field(field, table)
+                if table.metadata['datatype'] == 'composite':
+                    self.composites.add(field)
+
+            self.fields['ondevice'] = create_field('ondevice', None)
+
+    @read_api
+    def field_for(self, name, book_id, default_value=None):
+        '''
+        Return the value of the field ``name`` for the book identified by
+        ``book_id``. If no such book exists or it has no defined value for the
+        field ``name`` or no such field exists, then ``default_value`` is returned.
+
+        The returned value for is_multiple fields are always tuples.
+        '''
+        if self.composites and name in self.composites:
+            return self.composite_for(name, book_id,
+                    default_value=default_value)
+        try:
+            return self.fields[name].for_book(book_id, default_value=default_value)
+        except (KeyError, IndexError):
+            return default_value
+
+    @read_api
+    def composite_for(self, name, book_id, mi=None, default_value=''):
+        try:
+            f = self.fields[name]
+        except KeyError:
+            return default_value
+
+        if mi is None:
+            return f.get_value_with_cache(book_id, partial(self._get_metadata,
+                get_user_categories=False))
+        else:
+            return f.render_composite(book_id, mi)
+
+    @read_api
+    def field_ids_for(self, name, book_id):
+        '''
+        Return the ids (as a tuple) for the values that the field ``name`` has on the book
+        identified by ``book_id``. If there are no values, or no such book, or
+        no such field, an empty tuple is returned.
+        '''
+        try:
+            return self.fields[name].ids_for_book(book_id)
+        except (KeyError, IndexError):
+            return ()
+
+    @read_api
+    def books_for_field(self, name, item_id):
+        '''
+        Return all the books associated with the item identified by
+        ``item_id``, where the item belongs to the field ``name``.
+
+        Returned value is a tuple of book ids, or the empty tuple if the item
+        or the field does not exist.
+        '''
+        try:
+            return self.fields[name].books_for(item_id)
+        except (KeyError, IndexError):
+            return ()
+
+    @read_api
+    def all_book_ids(self):
+        '''
+        Frozen set of all known book ids.
+        '''
+        return frozenset(self.fields['uuid'].iter_book_ids())
+
+    @read_api
+    def all_field_ids(self, name):
+        '''
+        Frozen set of ids for all values in the field ``name``.
+        '''
+        return frozenset(iter(self.fields[name]))
+
+    @read_api
+    def author_data(self, author_id):
+        '''
+        Return author data as a dictionary with keys: name, sort, link
+
+        If no author with the specified id is found an empty dictionary is
+        returned.
+        '''
+        try:
+            return self.fields['authors'].author_data(author_id)
+        except (KeyError, IndexError):
+            return {}
+
+    @read_api
+    def format_metadata(self, book_id, fmt, allow_cache=True):
+        if not fmt:
+            return {}
+        fmt = fmt.upper()
+        if allow_cache:
+            x = self.format_metadata_cache[book_id].get(fmt, None)
+            if x is not None:
+                return x
+        try:
+            name = self.fields['formats'].format_fname(book_id, fmt)
+            path = self._field_for('path', book_id).replace('/', os.sep)
+        except:
+            return {}
+
+        ans = {}
+        if path and name:
+            ans = self.backend.format_metadata(book_id, fmt, name, path)
+            self.format_metadata_cache[book_id][fmt] = ans
+        return ans
+
+    @api
+    def get_metadata(self, book_id,
+            get_cover=False, get_user_categories=True, cover_as_data=False):
+        '''
+        Return metadata for the book identified by book_id as a :class:`Metadata` object.
+        Note that the list of formats is not verified. If get_cover is True,
+        the cover is returned, either a path to temp file as mi.cover or if
+        cover_as_data is True then as mi.cover_data.
+        '''
+
+        with self.read_lock:
+            mi = self._get_metadata(book_id, get_user_categories=get_user_categories)
+
+        if get_cover:
+            if cover_as_data:
+                cdata = self.cover(book_id)
+                if cdata:
+                    mi.cover_data = ('jpeg', cdata)
+            else:
+                mi.cover = self.cover(book_id, as_path=True)
+
+        return mi
+
+    @api
+    def cover(self, book_id,
+            as_file=False, as_image=False, as_path=False):
+        '''
+        Return the cover image or None. By default, returns the cover as a
+        bytestring.
+
+        WARNING: Using as_path will copy the cover to a temp file and return
+        the path to the temp file. You should delete the temp file when you are
+        done with it.
+
+        :param as_file: If True return the image as an open file object (a SpooledTemporaryFile)
+        :param as_image: If True return the image as a QImage object
+        :param as_path: If True return the image as a path pointing to a
+                        temporary file
+        '''
+        with self.read_lock:
+            try:
+                path = self._field_for('path', book_id).replace('/', os.sep)
+            except:
+                return None
+
+        with self.record_lock.lock(book_id):
+            return self.backend.cover(path, as_file=as_file, as_image=as_image,
+                    as_path=as_path)
+
+    @read_api
+    def multisort(self, fields):
+        all_book_ids = frozenset(self._all_book_ids())
+        get_metadata = partial(self._get_metadata, get_user_categories=False)
+
+        sort_keys = tuple(self.fields[field[0]].sort_keys_for_books(get_metadata,
+            all_book_ids) for field in fields)
+
+        if len(sort_keys) == 1:
+            sk = sort_keys[0]
+            return sorted(all_book_ids, key=lambda i:sk[i], reverse=not
+                    fields[1])
+        else:
+            return sorted(all_book_ids, key=partial(SortKey, fields, sort_keys))
+
+    # }}}
+
+class SortKey(object):
+
+    def __init__(self, fields, sort_keys, book_id):
+        self.orders = tuple(1 if f[1] else -1 for f in fields)
+        self.sort_key = tuple(sk[book_id] for sk in sort_keys)
+
+    def __cmp__(self, other):
+        for i, order in enumerate(self.orders):
+            ans = cmp(self.sort_key[i], other.sort_key[i])
+            if ans != 0:
+                return ans * order
+        return 0
+
+
+# Testing {{{
+
+def test(library_path):
+    from calibre.db.backend import DB
+    backend = DB(library_path)
+    cache = Cache(backend)
+    cache.init()
+    print ('All book ids:', cache.all_book_ids())
+
+if __name__ == '__main__':
+    from calibre.utils.config import prefs
+    test(prefs['library_path'])
+
+# }}}

src/calibre/db/fields.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+from future_builtins import map
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from threading import Lock
+
+from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
+from calibre.utils.icu import sort_key
+
+class Field(object):
+
+    def __init__(self, name, table):
+        self.name, self.table = name, table
+        self.has_text_data = self.metadata['datatype'] in ('text', 'comments',
+                'series', 'enumeration')
+        self.table_type = self.table.table_type
+        dt = self.metadata['datatype']
+        self._sort_key = (sort_key if dt == 'text' else lambda x: x)
+
+    @property
+    def metadata(self):
+        return self.table.metadata
+
+    def for_book(self, book_id, default_value=None):
+        '''
+        Return the value of this field for the book identified by book_id.
+        When no value is found, returns ``default_value``.
+        '''
+        raise NotImplementedError()
+
+    def ids_for_book(self, book_id):
+        '''
+        Return a tuple of items ids for items associated with the book
+        identified by book_ids. Returns an empty tuple if no such items are
+        found.
+        '''
+        raise NotImplementedError()
+
+    def books_for(self, item_id):
+        '''
+        Return the ids of all books associated with the item identified by
+        item_id as a tuple. An empty tuple is returned if no books are found.
+        '''
+        raise NotImplementedError()
+
+    def __iter__(self):
+        '''
+        Iterate over the ids for all values in this field
+        '''
+        raise NotImplementedError()
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        '''
+        Return a mapping of book_id -> sort_key. The sort key is suitable for
+        use in sorting the list of all books by this field, via the python cmp
+        method.
+        '''
+        raise NotImplementedError()
+
+
+class OneToOneField(Field):
+
+    def for_book(self, book_id, default_value=None):
+        return self.table.book_col_map.get(book_id, default_value)
+
+    def ids_for_book(self, book_id):
+        return (book_id,)
+
+    def books_for(self, item_id):
+        return (item_id,)
+
+    def __iter__(self):
+        return self.table.book_col_map.iterkeys()
+
+    def iter_book_ids(self):
+        return self.table.book_col_map.iterkeys()
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        return {id_ : self._sort_key(self.book_col_map.get(id_, '')) for id_ in
+                all_book_ids}
+
+class CompositeField(OneToOneField):
+
+    def __init__(self, *args, **kwargs):
+        OneToOneField.__init__(self, *args, **kwargs)
+
+        self._render_cache = {}
+        self._lock = Lock()
+
+    def render_composite(self, book_id, mi):
+        with self._lock:
+            ans = self._render_cache.get(book_id, None)
+        if ans is None:
+            ans = mi.get(self.metadata['label'])
+            with self._lock:
+                self._render_cache[book_id] = ans
+        return ans
+
+    def clear_cache(self):
+        with self._lock:
+            self._render_cache = {}
+
+    def pop_cache(self, book_id):
+        with self._lock:
+            self._render_cache.pop(book_id, None)
+
+    def get_value_with_cache(self, book_id, get_metadata):
+        with self._lock:
+            ans = self._render_cache.get(book_id, None)
+        if ans is None:
+            mi = get_metadata(book_id)
+            ans = mi.get(self.metadata['label'])
+        return ans
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        return {id_ : sort_key(self.get_value_with_cache(id_, get_metadata)) for id_ in
+                all_book_ids}
+
+
+class OnDeviceField(OneToOneField):
+
+    def __init__(self, name, table):
+        self.name = name
+        self.book_on_device_func = None
+
+    def book_on_device(self, book_id):
+        if callable(self.book_on_device_func):
+            return self.book_on_device_func(book_id)
+        return None
+
+    def set_book_on_device_func(self, func):
+        self.book_on_device_func = func
+
+    def for_book(self, book_id, default_value=None):
+        loc = []
+        count = 0
+        on = self.book_on_device(book_id)
+        if on is not None:
+            m, a, b, count = on[:4]
+            if m is not None:
+                loc.append(_('Main'))
+            if a is not None:
+                loc.append(_('Card A'))
+            if b is not None:
+                loc.append(_('Card B'))
+        return ', '.join(loc) + ((' (%s books)'%count) if count > 1 else '')
+
+    def __iter__(self):
+        return iter(())
+
+    def iter_book_ids(self):
+        return iter(())
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        return {id_ : self.for_book(id_) for id_ in
+                all_book_ids}
+
+class ManyToOneField(Field):
+
+    def for_book(self, book_id, default_value=None):
+        ids = self.table.book_col_map.get(book_id, None)
+        if ids is not None:
+            ans = self.id_map[ids]
+        else:
+            ans = default_value
+        return ans
+
+    def ids_for_book(self, book_id):
+        id_ = self.table.book_col_map.get(book_id, None)
+        if id_ is None:
+            return ()
+        return (id_,)
+
+    def books_for(self, item_id):
+        return self.table.col_book_map.get(item_id, ())
+
+    def __iter__(self):
+        return self.table.id_map.iterkeys()
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in
+                all_book_ids}
+        return {id_ : keys.get(
+            self.book_col_map.get(id_, None), '') for id_ in all_book_ids}
+
+class ManyToManyField(Field):
+
+    def __init__(self, *args, **kwargs):
+        Field.__init__(self, *args, **kwargs)
+        self.alphabetical_sort = self.name != 'authors'
+
+    def for_book(self, book_id, default_value=None):
+        ids = self.table.book_col_map.get(book_id, ())
+        if ids:
+            ans = tuple(self.id_map[i] for i in ids)
+        else:
+            ans = default_value
+        return ans
+
+    def ids_for_book(self, book_id):
+        return self.table.book_col_map.get(book_id, ())
+
+    def books_for(self, item_id):
+        return self.table.col_book_map.get(item_id, ())
+
+    def __iter__(self):
+        return self.table.id_map.iterkeys()
+
+    def sort_keys_for_books(self, get_metadata, all_book_ids):
+        keys = {id_ : self._sort_key(self.id_map.get(id_, '')) for id_ in
+                all_book_ids}
+
+        def sort_key_for_book(book_id):
+            item_ids = self.table.book_col_map.get(book_id, ())
+            if self.alphabetical_sort:
+                item_ids = sorted(item_ids, key=keys.get)
+            return tuple(map(keys.get, item_ids))
+
+        return {id_ : sort_key_for_book(id_) for id_ in all_book_ids}
+
+
+class AuthorsField(ManyToManyField):
+
+    def author_data(self, author_id):
+        return {
+            'name' : self.table.id_map[author_id],
+            'sort' : self.table.asort_map[author_id],
+            'link' : self.table.alink_map[author_id],
+        }
+
+class FormatsField(ManyToManyField):
+
+    def format_fname(self, book_id, fmt):
+        return self.table.fname_map[book_id][fmt.upper()]
+
+def create_field(name, table):
+    cls = {
+            ONE_ONE : OneToOneField,
+            MANY_ONE : ManyToOneField,
+            MANY_MANY : ManyToManyField,
+        }[table.table_type]
+    if name == 'authors':
+        cls = AuthorsField
+    elif name == 'ondevice':
+        cls = OnDeviceField
+    elif name == 'formats':
+        cls = FormatsField
+    elif table.metadata['datatype'] == 'composite':
+        cls = CompositeField
+    return cls(name, table)
+

src/calibre/db/locking.py

@@ -7,7 +7,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-from threading import Lock, Condition, current_thread
+from threading import Lock, Condition, current_thread, RLock
+from functools import partial
+from collections import Counter
 
 class LockingError(RuntimeError):
     pass
@@ -37,7 +39,7 @@ def create_locks():
     l = SHLock()
     return RWLockWrapper(l), RWLockWrapper(l, is_shared=False)
 
-class SHLock(object):
+class SHLock(object): # {{{
     '''
     Shareable lock class. Used to implement the Multiple readers-single writer
     paradigm. As best as I can tell, neither writer nor reader starvation
@@ -79,6 +81,11 @@ class SHLock(object):
                 return self._acquire_exclusive(blocking)
             assert not (self.is_shared and self.is_exclusive)
 
+    def owns_lock(self):
+        me = current_thread()
+        with self._lock:
+            return self._exclusive_owner is me or me in self._shared_owners
+
     def release(self):
         ''' Release the lock. '''
         #  This decrements the appropriate lock counters, and if the lock
@@ -189,6 +196,8 @@ class SHLock(object):
     def _return_waiter(self, waiter):
         self._free_waiters.append(waiter)
 
+# }}}
+
 class RWLockWrapper(object):
 
     def __init__(self, shlock, is_shared=True):
@@ -200,16 +209,124 @@ class RWLockWrapper(object):
         return self
 
     def __exit__(self, *args):
+        self.release()
+
+    def release(self):
         self._shlock.release()
 
+    def owns_lock(self):
+        return self._shlock.owns_lock()
+
+class RecordLock(object):
+
+    '''
+    Lock records identified by hashable ids. To use
+
+    rl = RecordLock()
+
+    with rl.lock(some_id):
+        # do something
+
+    This will lock the record identified by some_id exclusively. The lock is
+    recursive, which means that you can lock the same record multiple times in
+    the same thread.
+
+    This class co-operates with the SHLock class. If you try to lock a record
+    in a thread that already holds the SHLock, a LockingError is raised. This
+    is to prevent the possibility of a cross-lock deadlock.
+
+    A cross-lock deadlock is still possible if you first lock a record and then
+    acquire the SHLock, but the usage pattern for this lock makes this highly
+    unlikely (this lock should be acquired immediately before any file I/O on
+    files in the library and released immediately after).
+    '''
+
+    class Wrap(object):
+
+        def __init__(self, release):
+            self.release = release
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args, **kwargs):
+            self.release()
+            self.release = None
+
+    def __init__(self, sh_lock):
+        self._lock = Lock()
+        #  This is for recycling lock objects.
+        self._free_locks = [RLock()]
+        self._records = {}
+        self._counter = Counter()
+        self.sh_lock = sh_lock
+
+    def lock(self, record_id):
+        if self.sh_lock.owns_lock():
+            raise LockingError('Current thread already holds a shared lock,'
+                ' you cannot also ask for record lock as this could cause a'
+                ' deadlock.')
+        with self._lock:
+            l = self._records.get(record_id, None)
+            if l is None:
+                l = self._take_lock()
+                self._records[record_id] = l
+            self._counter[record_id] += 1
+        l.acquire()
+        return RecordLock.Wrap(partial(self.release, record_id))
+
+    def release(self, record_id):
+        with self._lock:
+            l = self._records.pop(record_id, None)
+            if l is None:
+                raise LockingError('No lock acquired for record %r'%record_id)
+            l.release()
+            self._counter[record_id] -= 1
+            if self._counter[record_id] > 0:
+                self._records[record_id] = l
+            else:
+                self._return_lock(l)
+
+    def _take_lock(self):
+        try:
+            return self._free_locks.pop()
+        except IndexError:
+            return RLock()
+
+    def _return_lock(self, lock):
+        self._free_locks.append(lock)
 
 # Tests {{{
 if __name__ == '__main__':
     import time, random, unittest
     from threading import Thread
 
-    class TestSHLock(unittest.TestCase):
-        """Testcases for SHLock class."""
+    class TestLock(unittest.TestCase):
+        """Testcases for Lock classes."""
+
+        def test_owns_locks(self):
+            lock = SHLock()
+            self.assertFalse(lock.owns_lock())
+            lock.acquire(shared=True)
+            self.assertTrue(lock.owns_lock())
+            lock.release()
+            self.assertFalse(lock.owns_lock())
+            lock.acquire(shared=False)
+            self.assertTrue(lock.owns_lock())
+            lock.release()
+            self.assertFalse(lock.owns_lock())
+
+            done = []
+            def test():
+                if not lock.owns_lock():
+                    done.append(True)
+            lock.acquire()
+            t = Thread(target=test)
+            t.daemon = True
+            t.start()
+            t.join(1)
+            self.assertEqual(len(done), 1)
+            lock.release()
 
         def test_multithread_deadlock(self):
             lock = SHLock()
@@ -345,8 +462,38 @@ if __name__ == '__main__':
             self.assertFalse(lock.is_shared)
             self.assertFalse(lock.is_exclusive)
 
+        def test_record_lock(self):
+            shlock = SHLock()
+            lock = RecordLock(shlock)
 
-    suite = unittest.TestLoader().loadTestsFromTestCase(TestSHLock)
+            shlock.acquire()
+            self.assertRaises(LockingError, lock.lock, 1)
+            shlock.release()
+            with lock.lock(1):
+                with lock.lock(1):
+                    pass
+
+            def dolock():
+                with lock.lock(1):
+                    time.sleep(0.1)
+
+            t = Thread(target=dolock)
+            t.daemon = True
+            with lock.lock(1):
+                t.start()
+                t.join(0.2)
+                self.assertTrue(t.is_alive())
+            t.join(0.11)
+            self.assertFalse(t.is_alive())
+
+            t = Thread(target=dolock)
+            t.daemon = True
+            with lock.lock(2):
+                t.start()
+                t.join(0.11)
+                self.assertFalse(t.is_alive())
+
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestLock)
     unittest.TextTestRunner(verbosity=2).run(suite)
 
 # }}}

src/calibre/db/tables.py

@@ -17,6 +17,8 @@ from calibre.ebooks.metadata import author_to_author_sort
 
 _c_speedup = plugins['speedup'][0]
 
+ONE_ONE, MANY_ONE, MANY_MANY = xrange(3)
+
 def _c_convert_timestamp(val):
     if not val:
         return None
@@ -57,6 +59,8 @@ class OneToOneTable(Table):
     timestamp, size, etc.
     '''
 
+    table_type = ONE_ONE
+
     def read(self, db):
         self.book_col_map = {}
         idcol = 'id' if self.metadata['table'] == 'books' else 'book'
@@ -73,6 +77,17 @@ class SizeTable(OneToOneTable):
                 'WHERE data.book=books.id) FROM books'):
             self.book_col_map[row[0]] = self.unserialize(row[1])
 
+class CompositeTable(OneToOneTable):
+
+    def read(self, db):
+        self.book_col_map = {}
+        d = self.metadata['display']
+        self.composite_template = ['composite_template']
+        self.contains_html = d['contains_html']
+        self.make_category = d['make_category']
+        self.composite_sort = d['composite_sort']
+        self.use_decorations = d['use_decorations']
+
 class ManyToOneTable(Table):
 
     '''
@@ -82,9 +97,10 @@ class ManyToOneTable(Table):
     Each book however has only one value for data of this type.
     '''
 
+    table_type = MANY_ONE
+
     def read(self, db):
         self.id_map = {}
-        self.extra_map = {}
         self.col_book_map = {}
         self.book_col_map = {}
         self.read_id_maps(db)
@@ -105,6 +121,9 @@ class ManyToOneTable(Table):
             self.col_book_map[row[1]].append(row[0])
             self.book_col_map[row[0]] = row[1]
 
+        for key in tuple(self.col_book_map.iterkeys()):
+            self.col_book_map[key] = tuple(self.col_book_map[key])
+
 class ManyToManyTable(ManyToOneTable):
 
     '''
@@ -113,6 +132,8 @@ class ManyToManyTable(ManyToOneTable):
     book. For example: tags or authors.
     '''
 
+    table_type = MANY_MANY
+
     def read_maps(self, db):
         for row in db.conn.execute(
                 'SELECT book, {0} FROM {1}'.format(
@@ -124,14 +145,21 @@ class ManyToManyTable(ManyToOneTable):
                 self.book_col_map[row[0]] = []
             self.book_col_map[row[0]].append(row[1])
 
+        for key in tuple(self.col_book_map.iterkeys()):
+            self.col_book_map[key] = tuple(self.col_book_map[key])
+
+        for key in tuple(self.book_col_map.iterkeys()):
+            self.book_col_map[key] = tuple(self.book_col_map[key])
+
 class AuthorsTable(ManyToManyTable):
 
     def read_id_maps(self, db):
         self.alink_map = {}
+        self.asort_map  = {}
         for row in db.conn.execute(
                 'SELECT id, name, sort, link FROM authors'):
             self.id_map[row[0]] = row[1]
-            self.extra_map[row[0]] = (row[2] if row[2] else
+            self.asort_map[row[0]] = (row[2] if row[2] else
                     author_to_author_sort(row[1]))
             self.alink_map[row[0]] = row[3]
 
@@ -141,14 +169,25 @@ class FormatsTable(ManyToManyTable):
         pass
 
     def read_maps(self, db):
+        self.fname_map = {}
         for row in db.conn.execute('SELECT book, format, name FROM data'):
             if row[1] is not None:
-                if row[1] not in self.col_book_map:
-                    self.col_book_map[row[1]] = []
-                self.col_book_map[row[1]].append(row[0])
+                fmt = row[1].upper()
+                if fmt not in self.col_book_map:
+                    self.col_book_map[fmt] = []
+                self.col_book_map[fmt].append(row[0])
                 if row[0] not in self.book_col_map:
                     self.book_col_map[row[0]] = []
-                self.book_col_map[row[0]].append((row[1], row[2]))
+                self.book_col_map[row[0]].append(fmt)
+                if row[0] not in self.fname_map:
+                    self.fname_map[row[0]] = {}
+                self.fname_map[row[0]][fmt] = row[2]
+
+        for key in tuple(self.col_book_map.iterkeys()):
+            self.col_book_map[key] = tuple(self.col_book_map[key])
+
+        for key in tuple(self.book_col_map.iterkeys()):
+            self.book_col_map[key] = tuple(self.book_col_map[key])
 
 class IdentifiersTable(ManyToManyTable):
 
@@ -162,6 +201,9 @@ class IdentifiersTable(ManyToManyTable):
                     self.col_book_map[row[1]] = []
                 self.col_book_map[row[1]].append(row[0])
                 if row[0] not in self.book_col_map:
-                    self.book_col_map[row[0]] = []
-                self.book_col_map[row[0]].append((row[1], row[2]))
+                    self.book_col_map[row[0]] = {}
+                self.book_col_map[row[0]][row[1]] = row[2]
+
+        for key in tuple(self.col_book_map.iterkeys()):
+            self.col_book_map[key] = tuple(self.col_book_map[key])
 

src/calibre/db/view.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+def sanitize_sort_field_name(field_metadata, field):
+    field = field_metadata.search_term_to_field_key(field.lower().strip())
+    # translate some fields to their hidden equivalent
+    field = {'title': 'sort', 'authors':'author_sort'}.get(field, field)
+    return field
+
+class View(object):
+
+    def __init__(self, cache):
+        self.cache = cache
+        self.marked_ids = {}
+        self._field_getters = {}
+        for col, idx in cache.backend.FIELD_MAP.iteritems():
+            if isinstance(col, int):
+                label = self.cache.backend.custom_column_num_map[col]['label']
+                label = (self.cache.backend.field_metadata.custom_field_prefix
+                        + label)
+                self._field_getters[idx] = partial(self.get, label)
+            else:
+                try:
+                    self._field_getters[idx] = {
+                        'id'      : self._get_id,
+                        'au_map'  : self.get_author_data,
+                        'ondevice': self.get_ondevice,
+                        'marked'  : self.get_marked,
+                    }[col]
+                except KeyError:
+                    self._field_getters[idx] = partial(self.get, col)
+
+        self._map = list(self.cache.all_book_ids())
+        self._map_filtered = list(self._map)
+
+    @property
+    def field_metadata(self):
+        return self.cache.field_metadata
+
+    def _get_id(self, idx, index_is_id=True):
+        ans = idx if index_is_id else self.index_to_id(idx)
+        return ans
+
+    def get_field_map_field(self, row, col, index_is_id=True):
+        '''
+        Supports the legacy FIELD_MAP interface for getting metadata. Do not use
+        in new code.
+        '''
+        getter = self._field_getters[col]
+        return getter(row, index_is_id=index_is_id)
+
+    def index_to_id(self, idx):
+        return self._map_filtered[idx]
+
+    def get(self, field, idx, index_is_id=True, default_value=None):
+        id_ = idx if index_is_id else self.index_to_id(idx)
+        return self.cache.field_for(field, id_)
+
+    def get_ondevice(self, idx, index_is_id=True, default_value=''):
+        id_ = idx if index_is_id else self.index_to_id(idx)
+        self.cache.field_for('ondevice', id_, default_value=default_value)
+
+    def get_marked(self, idx, index_is_id=True, default_value=None):
+        id_ = idx if index_is_id else self.index_to_id(idx)
+        return self.marked_ids.get(id_, default_value)
+
+    def get_author_data(self, idx, index_is_id=True, default_value=()):
+        '''
+        Return author data for all authors of the book identified by idx as a
+        tuple of dictionaries. The dictionaries should never be empty, unless
+        there is a bug somewhere. The list could be empty if idx point to an
+        non existent book, or book with no authors (though again a book with no
+        authors should never happen).
+
+        Each dictionary has the keys: name, sort, link. Link can be an empty
+        string.
+
+        default_value is ignored, this method always returns a tuple
+        '''
+        id_ = idx if index_is_id else self.index_to_id(idx)
+        with self.cache.read_lock:
+            ids = self.cache._field_ids_for('authors', id_)
+            ans = []
+            for id_ in ids:
+                ans.append(self.cache._author_data(id_))
+        return tuple(ans)
+
+    def multisort(self, fields=[], subsort=False):
+        fields = [(sanitize_sort_field_name(self.field_metadata, x), bool(y)) for x, y in fields]
+        keys = self.field_metadata.sortable_field_keys()
+        fields = [x for x in fields if x[0] in keys]
+        if subsort and 'sort' not in [x[0] for x in fields]:
+            fields += [('sort', True)]
+        if not fields:
+            fields = [('timestamp', False)]
+
+        sorted_book_ids = self.cache.multisort(fields)
+        sorted_book_ids
+        # TODO: change maps
+
+

src/calibre/devices/android/driver.py

@@ -39,7 +39,7 @@ class ANDROID(USBMS):
             0x22b8 : { 0x41d9 : [0x216], 0x2d61 : [0x100], 0x2d67 : [0x100],
                        0x41db : [0x216], 0x4285 : [0x216], 0x42a3 : [0x216],
                        0x4286 : [0x216], 0x42b3 : [0x216], 0x42b4 : [0x216],
-                       0x7086 : [0x0226], 0x70a8: [0x9999],
+                       0x7086 : [0x0226], 0x70a8: [0x9999], 0x42c4 : [0x216],
                      },
 
             # Sony Ericsson
@@ -60,6 +60,7 @@ class ANDROID(USBMS):
                        0x685e : [0x0400],
                        0x6860 : [0x0400],
                        0x6877 : [0x0400],
+                       0x689e : [0x0400],
                      },
 
             # Viewsonic
@@ -124,7 +125,8 @@ class ANDROID(USBMS):
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
             'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK',
-            'MB525', 'ANDROID2.3', 'SGH-I997']
+            'MB525', 'ANDROID2.3', 'SGH-I997', 'GT-I5800_CARD', 'MB612',
+            'GT-S5830_CARD']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

src/calibre/devices/kobo/driver.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import os
 import sqlite3 as sqlite
+from contextlib import closing
 
 from calibre.devices.usbms.books import BookList
 from calibre.devices.kobo.books import Book
@@ -22,7 +23,7 @@ class KOBO(USBMS):
     gui_name = 'Kobo Reader'
     description = _('Communicate with the Kobo Reader')
     author = 'Timothy Legge'
-    version = (1, 0, 9)
+    version = (1, 0, 10)
 
     dbversion = 0
     fwversion = 0
@@ -48,12 +49,16 @@ class KOBO(USBMS):
 
     VIRTUAL_BOOK_EXTENSIONS = frozenset(['kobo'])
 
-    EXTRA_CUSTOMIZATION_MESSAGE = _('The Kobo supports only one collection '
-            'currently: the \"Im_Reading\" list.  Create a tag called \"Im_Reading\" ')+\
-                    'for automatic management'
+    EXTRA_CUSTOMIZATION_MESSAGE = [
+            _('The Kobo supports several collections including ')+\
+                    'Read, Closed, Im_Reading ' +\
+            _('Create tags for automatic management'),
+    ]
 
     EXTRA_CUSTOMIZATION_DEFAULT = ', '.join(['tags'])
 
+    OPT_COLLECTIONS = 0
+
     def initialize(self):
         USBMS.initialize(self)
         self.book_class = Book
@@ -188,7 +193,9 @@ class KOBO(USBMS):
                 traceback.print_exc()
             return changed
 
-        connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
+        with closing(sqlite.connect(
+            self.normalize_path(self._main_prefix +
+                '.kobo/KoboReader.sqlite'))) as connection:
 
             # return bytestrings if the content cannot the decoded as unicode
             connection.text_factory = lambda x: unicode(x, "utf-8", "ignore")
@@ -258,7 +265,6 @@ class KOBO(USBMS):
                     need_sync = True
 
             cursor.close()
-        connection.close()
 
         # Remove books that are no longer in the filesystem. Cache contains
         # indices into the booklist if book not in filesystem, None otherwise
@@ -288,7 +294,8 @@ class KOBO(USBMS):
         #    2) content
 
         debug_print('delete_via_sql: ContentID: ', ContentID, 'ContentType: ', ContentType)
-        connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
+        with closing(sqlite.connect(self.normalize_path(self._main_prefix +
+            '.kobo/KoboReader.sqlite'))) as connection:
 
             # return bytestrings if the content cannot the decoded as unicode
             connection.text_factory = lambda x: unicode(x, "utf-8", "ignore")
@@ -337,7 +344,6 @@ class KOBO(USBMS):
                 print "Error condition ImageID was not found"
                 print "You likely tried to delete a book that the kobo has not yet added to the database"
 
-        connection.close()
         # If all this succeeds we need to delete the images files via the ImageID
         return ImageID
 
@@ -664,7 +670,8 @@ class KOBO(USBMS):
         # Needs to be outside books collection as in the case of removing
         # the last book from the collection the list of books is empty
         # and the removal of the last book would not occur
-        connection = sqlite.connect(self.normalize_path(self._main_prefix + '.kobo/KoboReader.sqlite'))
+        with closing(sqlite.connect(self.normalize_path(self._main_prefix +
+            '.kobo/KoboReader.sqlite'))) as connection:
 
             # return bytestrings if the content cannot the decoded as unicode
             connection.text_factory = lambda x: unicode(x, "utf-8", "ignore")
@@ -693,10 +700,10 @@ class KOBO(USBMS):
                             if category in readstatuslist.keys():
                                 # Manage ReadStatus
                                 self.set_readstatus(connection, ContentID, readstatuslist.get(category))
-                        if category == 'Shortlist' and self.dbversion >= 14:
+                            elif category == 'Shortlist' and self.dbversion >= 14:
                                 # Manage FavouritesIndex/Shortlist
                                 self.set_favouritesindex(connection, ContentID)
-                        if category in accessibilitylist.keys():
+                            elif category in accessibilitylist.keys():
                                 # Do not manage the Accessibility List
                                 pass
             else: # No collections
@@ -707,8 +714,6 @@ class KOBO(USBMS):
                     debug_print("No Collections - reseting FavouritesIndex")
                     self.reset_favouritesindex(connection, oncard)
 
-        connection.close()
-
 #        debug_print('Finished update_device_database_collections', collections_attributes)
 
     def sync_booklists(self, booklists, end_session=True):
@@ -723,7 +728,7 @@ class KOBO(USBMS):
         opts = self.settings()
         if opts.extra_customization:
             collections = [x.lower().strip() for x in
-                    opts.extra_customization.split(',')]
+                    opts.extra_customization[self.OPT_COLLECTIONS].split(',')]
         else:
             collections = []
 

src/calibre/devices/usbms/device.py

@@ -1077,8 +1077,13 @@ class Device(DeviceConfig, DevicePlugin):
         settings = self.settings()
         template = self.save_template()
         if mdata.tags and _('News') in mdata.tags:
+            try:
+                p = mdata.pubdate
+                date  = (p.year, p.month, p.day)
+            except:
                 today = time.localtime()
-            template = "{title}_%d-%d-%d" % (today[0], today[1], today[2])
+                date = (today[0], today[1], today[2])
+            template = "{title}_%d-%d-%d" % date
         use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
 
         fname = sanitize(fname)

src/calibre/devices/usbms/driver.py

@@ -94,11 +94,29 @@ class USBMS(CLI, Device):
         self.report_progress(1.0, _('Get device information...'))
         self.driveinfo = {}
         if self._main_prefix is not None:
+            try:
                 self.driveinfo['main'] = self._update_driveinfo_file(self._main_prefix, 'main')
+            except (IOError, OSError) as e:
+                raise IOError(_('Failed to access files in the main memory of'
+                        ' your device. You should contact the device'
+                        ' manufacturer for support. Common fixes are:'
+                        ' try a different USB cable/USB port on your computer.'
+                        ' If you device has a "Reset to factory defaults" type'
+                        ' of setting somewhere, use it. Underlying error: %s')
+                        % e)
+        try:
             if self._card_a_prefix is not None:
                 self.driveinfo['A'] = self._update_driveinfo_file(self._card_a_prefix, 'A')
             if self._card_b_prefix is not None:
                 self.driveinfo['B'] = self._update_driveinfo_file(self._card_b_prefix, 'B')
+        except (IOError, OSError) as e:
+            raise IOError(_('Failed to access files on the SD card in your'
+                ' device. This can happen for many reasons. The SD card may be'
+                ' corrupted, it may be too large for your device, it may be'
+                ' write-protected, etc. Try a different SD card, or reformat'
+                ' your SD card using the FAT32 filesystem. Also make sure'
+                ' there are not too many files in the root of your SD card.'
+                ' Underlying error: %s') % e)
         return (self.get_gui_name(), '', '', '', self.driveinfo)
 
     def set_driveinfo_name(self, location_code, name):

src/calibre/ebooks/__init__.py

@@ -159,7 +159,7 @@ def normalize(x):
     return x
 
 def calibre_cover(title, author_string, series_string=None,
-        output_format='jpg', title_size=46, author_size=36):
+        output_format='jpg', title_size=46, author_size=36, logo_path=None):
     title = normalize(title)
     author_string = normalize(author_string)
     series_string = normalize(series_string)
@@ -167,7 +167,9 @@ def calibre_cover(title, author_string, series_string=None,
     lines = [TextLine(title, title_size), TextLine(author_string, author_size)]
     if series_string:
         lines.append(TextLine(series_string, author_size))
-    return create_cover_page(lines, I('library.png'), output_format='jpg')
+    if logo_path is None:
+        logo_path = I('library.png')
+    return create_cover_page(lines, logo_path, output_format='jpg')
 
 UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$')
 

src/calibre/ebooks/chardet/__init__.py

@@ -38,8 +38,12 @@ ENCODING_PATS = [
 ENTITY_PATTERN = re.compile(r'&(\S+?);')
 
 def strip_encoding_declarations(raw):
+    limit = 50*1024
     for pat in ENCODING_PATS:
-        raw = pat.sub('', raw)
+        prefix = raw[:limit]
+        suffix = raw[limit:]
+        prefix = pat.sub('', prefix)
+        raw = prefix + suffix
     return raw
 
 def substitute_entites(raw):

src/calibre/ebooks/conversion/cli.py

@@ -137,7 +137,9 @@ def add_pipeline_options(parser, plumber):
                       'extra_css', 'smarten_punctuation',
                       'margin_top', 'margin_left', 'margin_right',
                       'margin_bottom', 'change_justification',
-                      'insert_blank_line', 'remove_paragraph_spacing','remove_paragraph_spacing_indent_size',
+                      'insert_blank_line', 'insert_blank_line_size',
+                      'remove_paragraph_spacing',
+                      'remove_paragraph_spacing_indent_size',
                       'asciiize',
                   ]
                   ),

src/calibre/ebooks/conversion/plumber.py

@@ -366,9 +366,9 @@ OptionRecommendation(name='remove_paragraph_spacing',
 
 OptionRecommendation(name='remove_paragraph_spacing_indent_size',
         recommended_value=1.5, level=OptionRecommendation.LOW,
-        help=_('When calibre removes inter paragraph spacing, it automatically '
+        help=_('When calibre removes blank lines between paragraphs, it automatically '
             'sets a paragraph indent, to ensure that paragraphs can be easily '
-            'distinguished. This option controls the width of that indent.')
+            'distinguished. This option controls the width of that indent (in em).')
         ),
 
 OptionRecommendation(name='prefer_metadata_cover',
@@ -384,6 +384,13 @@ OptionRecommendation(name='insert_blank_line',
             )
         ),
 
+OptionRecommendation(name='insert_blank_line_size',
+        recommended_value=0.5, level=OptionRecommendation.LOW,
+        help=_('Set the height of the inserted blank lines (in em).'
+            ' The height of the lines between paragraphs will be twice the value'
+            ' set here.')
+        ),
+
 OptionRecommendation(name='remove_first_image',
         recommended_value=False, level=OptionRecommendation.LOW,
         help=_('Remove the first image from the input ebook. Useful if the '
@@ -602,7 +609,7 @@ OptionRecommendation(name='sr3_replace',
         input_fmt = os.path.splitext(self.input)[1]
         if not input_fmt:
             raise ValueError('Input file must have an extension')
-        input_fmt = input_fmt[1:].lower()
+        input_fmt = input_fmt[1:].lower().replace('original_', '')
         self.archive_input_tdir = None
         if input_fmt in ARCHIVE_FMTS:
             self.log('Processing archive...')
@@ -1048,6 +1055,7 @@ OptionRecommendation(name='sr3_replace',
         with self.output_plugin:
             self.output_plugin.convert(self.oeb, self.output, self.input_plugin,
                 self.opts, self.log)
+        self.oeb.clean_temp_files()
         self.ui_reporter(1.)
         run_plugins_on_postprocess(self.output, self.output_fmt)
 

src/calibre/ebooks/conversion/preprocess.py

@@ -303,6 +303,9 @@ class CSSPreProcessor(object):
 class HTMLPreProcessor(object):
 
     PREPROCESS = [
+                  # Remove huge block of contiguous spaces as they slow down
+                  # the following regexes pretty badly
+                  (re.compile(r'\s{10000,}'), lambda m: ''),
                   # Some idiotic HTML generators (Frontpage I'm looking at you)
                   # Put all sorts of crap into <head>. This messes up lxml
                   (re.compile(r'<head[^>]*>\n*(.*?)\n*</head>', re.IGNORECASE|re.DOTALL),

src/calibre/ebooks/htmlz/input.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import os
 
-from calibre import guess_type, walk
+from calibre import guess_type
 from calibre.customize.conversion import InputFormatPlugin
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.metadata.opf2 import OPF
@@ -25,16 +25,50 @@ class HTMLZInput(InputFormatPlugin):
                 accelerators):
         self.log = log
         html = u''
+        top_levels = []
 
         # Extract content from zip archive.
         zf = ZipFile(stream)
         zf.extractall()
 
-        for x in walk('.'):
-            if os.path.splitext(x)[1].lower() in ('.html', '.xhtml', '.htm'):
-                with open(x, 'rb') as tf:
-                    html = tf.read()
+        # Find the HTML file in the archive. It needs to be
+        # top level.
+        index = u''
+        multiple_html = False
+        # Get a list of all top level files in the archive.
+        for x in os.listdir('.'):
+            if os.path.isfile(x):
+                top_levels.append(x)
+        # Try to find an index. file.
+        for x in top_levels:
+            if x.lower() in ('index.html', 'index.xhtml', 'index.htm'):
+                index = x
                 break
+        # Look for multiple HTML files in the archive. We look at the
+        # top level files only as only they matter in HTMLZ.
+        for x in top_levels:
+            if os.path.splitext(x)[1].lower() in ('.html', '.xhtml', '.htm'):
+                # Set index to the first HTML file found if it's not
+                # called index.
+                if not index:
+                    index = x
+                else:
+                    multiple_html = True
+        # Warn the user if there multiple HTML file in the archive. HTMLZ
+        # supports a single HTML file. A conversion with a multiple HTML file
+        # HTMLZ archive probably won't turn out as the user expects. With
+        # Multiple HTML files ZIP input should be used in place of HTMLZ.
+        if multiple_html:
+            log.warn(_('Multiple HTML files found in the archive. Only %s will be used.') % index)
+
+        if index:
+            with open(index, 'rb') as tf:
+                html = tf.read()
+        else:
+            raise Exception(_('No top level HTML file found.'))
+
+        if not html:
+            raise Exception(_('Top level HTML file %s is empty') % index)
 
         # Encoding
         if options.input_encoding:
@@ -75,7 +109,7 @@ class HTMLZInput(InputFormatPlugin):
         # Get the cover path from the OPF.
         cover_path = None
         opf = None
-        for x in walk('.'):
+        for x in top_levels:
             if os.path.splitext(x)[1].lower() in ('.opf'):
                 opf = x
                 break

src/calibre/ebooks/metadata/book/base.py

@@ -742,7 +742,7 @@ class Metadata(object):
         ans += [('ISBN', unicode(self.isbn))]
         ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
         if self.series:
-            ans += [_('Series'), unicode(self.series) + ' #%s'%self.format_series_index()]
+            ans += [(_('Series'), unicode(self.series) + ' #%s'%self.format_series_index())]
         ans += [(_('Language'), unicode(self.language))]
         if self.timestamp is not None:
             ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]

src/calibre/ebooks/mobi/debug.py

@@ -7,10 +7,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import struct, datetime
+import struct, datetime, sys, os
+from collections import OrderedDict
 from calibre.utils.date import utc_tz
 from calibre.ebooks.mobi.langcodes import main_language, sub_language
+from calibre.ebooks.mobi.writer2.utils import decode_hex_number, decint
 
+# PalmDB {{{
 class PalmDOCAttributes(object):
 
     class Attr(object):
@@ -94,8 +97,9 @@ class PalmDB(object):
         ans.append('Number of records: %s'%self.number_of_records)
 
         return '\n'.join(ans)
+# }}}
 
-class Record(object):
+class Record(object): # {{{
 
     def __init__(self, raw, header):
         self.offset, self.flags, self.uid = header
@@ -103,9 +107,11 @@ class Record(object):
 
     @property
     def header(self):
-        return 'Offset: %d Flags: %d UID: %d'%(self.offset, self.flags,
-                self.uid)
+        return 'Offset: %d Flags: %d UID: %d First 4 bytes: %r Size: %d'%(self.offset, self.flags,
+                self.uid, self.raw[:4], len(self.raw))
+# }}}
 
+# EXTH {{{
 class EXTHRecord(object):
 
     def __init__(self, type_, data):
@@ -189,9 +195,9 @@ class EXTHHeader(object):
         for r in self.records:
             ans.append(str(r))
         return '\n'.join(ans)
+# }}}
 
-
-class MOBIHeader(object):
+class MOBIHeader(object): # {{{
 
     def __init__(self, record0):
         self.raw = record0.raw
@@ -271,6 +277,7 @@ class MOBIHeader(object):
             self.drm_flags = bin(struct.unpack(b'>I', self.raw[176:180])[0])
         self.has_extra_data_flags = self.length >= 232 and len(self.raw) >= 232+16
         self.has_fcis_flis = False
+        self.has_multibytes = self.has_indexing_bytes = self.has_uncrossable_breaks = False
         if self.has_extra_data_flags:
             self.unknown4 = self.raw[180:192]
             self.first_content_record, self.last_content_record = \
@@ -280,8 +287,11 @@ class MOBIHeader(object):
                     self.flis_count) = struct.unpack(b'>IIII',
                             self.raw[200:216])
             self.unknown6 = self.raw[216:240]
-            self.extra_data_flags = bin(struct.unpack(b'>I',
-                self.raw[240:244])[0])
+            self.extra_data_flags = struct.unpack(b'>I',
+                self.raw[240:244])[0]
+            self.has_multibytes = bool(self.extra_data_flags & 0b1)
+            self.has_indexing_bytes = bool(self.extra_data_flags & 0b10)
+            self.has_uncrossable_breaks = bool(self.extra_data_flags & 0b100)
             self.primary_index_record, = struct.unpack(b'>I',
                     self.raw[244:248])
 
@@ -311,7 +321,8 @@ class MOBIHeader(object):
         ans.append('Secondary index record: %d (null val: %d)'%(
             self.secondary_index_record, 0xffffffff))
         ans.append('Reserved2: %r'%self.reserved2)
-        ans.append('First non-book record: %d'% self.first_non_book_record)
+        ans.append('First non-book record (null value: %d): %d'%(0xffffffff,
+            self.first_non_book_record))
         ans.append('Full name offset: %d'%self.fullname_offset)
         ans.append('Full name length: %d bytes'%self.fullname_length)
         ans.append('Langcode: %r'%self.locale_raw)
@@ -341,8 +352,12 @@ class MOBIHeader(object):
             ans.append('FLIS number: %d'% self.flis_number)
             ans.append('FLIS count: %d'% self.flis_count)
             ans.append('Unknown6: %r'% self.unknown6)
-            ans.append('Extra data flags: %r'%self.extra_data_flags)
-            ans.append('Primary index record: %d'%self.primary_index_record)
+            ans.append(('Extra data flags: %s (has multibyte: %s) '
+                '(has indexing: %s) (has uncrossable breaks: %s)')%(
+                    bin(self.extra_data_flags), self.has_multibytes,
+                    self.has_indexing_bytes, self.has_uncrossable_breaks ))
+            ans.append('Primary index record (null value: %d): %d'%(0xffffffff,
+                self.primary_index_record))
 
         ans = '\n'.join(ans)
 
@@ -355,8 +370,345 @@ class MOBIHeader(object):
 
         ans += '\nRecord 0 length: %d'%len(self.raw)
         return ans
+# }}}
 
-class MOBIFile(object):
+class TagX(object): # {{{
+
+    def __init__(self, raw, control_byte_count):
+        self.tag = ord(raw[0])
+        self.num_values = ord(raw[1])
+        self.bitmask = ord(raw[2])
+        # End of file = 1 iff last entry
+        # When it is 1 all others are 0
+        self.eof = ord(raw[3])
+
+        self.is_eof = (self.eof == 1 and self.tag == 0 and self.num_values == 0
+                and self.bitmask == 0)
+
+    def __repr__(self):
+        return 'TAGX(tag=%02d, num_values=%d, bitmask=%r, eof=%d)' % (self.tag,
+                self.num_values, bin(self.bitmask), self.eof)
+    # }}}
+
+class IndexHeader(object): # {{{
+
+    def __init__(self, record):
+        self.record = record
+        raw = self.record.raw
+        if raw[:4] != b'INDX':
+            raise ValueError('Invalid Primary Index Record')
+
+        self.header_length, = struct.unpack('>I', raw[4:8])
+        self.unknown1 = raw[8:16]
+        self.index_type, = struct.unpack('>I', raw[16:20])
+        self.index_type_desc = {0: 'normal', 2:
+                'inflection'}.get(self.index_type, 'unknown')
+        self.idxt_start, = struct.unpack('>I', raw[20:24])
+        self.index_count, = struct.unpack('>I', raw[24:28])
+        self.index_encoding_num, = struct.unpack('>I', raw[28:32])
+        self.index_encoding = {65001: 'utf-8', 1252:
+                'cp1252'}.get(self.index_encoding_num, 'unknown')
+        if self.index_encoding == 'unknown':
+            raise ValueError(
+                'Unknown index encoding: %d'%self.index_encoding_num)
+        self.locale_raw, = struct.unpack(b'>I', raw[32:36])
+        langcode = self.locale_raw
+        langid    = langcode & 0xFF
+        sublangid = (langcode >> 10) & 0xFF
+        self.language = main_language.get(langid, 'ENGLISH')
+        self.sublanguage = sub_language.get(sublangid, 'NEUTRAL')
+        self.num_index_entries, = struct.unpack('>I', raw[36:40])
+        self.ordt_start, = struct.unpack('>I', raw[40:44])
+        self.ligt_start, = struct.unpack('>I', raw[44:48])
+        self.num_of_ligt_entries, = struct.unpack('>I', raw[48:52])
+        self.num_of_cncx_blocks, = struct.unpack('>I', raw[52:56])
+        self.unknown2 = raw[56:180]
+        self.tagx_offset, = struct.unpack(b'>I', raw[180:184])
+        if self.tagx_offset != self.header_length:
+            raise ValueError('TAGX offset and header length disagree')
+        self.unknown3 = raw[184:self.header_length]
+
+        tagx = raw[self.header_length:]
+        if not tagx.startswith(b'TAGX'):
+            raise ValueError('Invalid TAGX section')
+        self.tagx_header_length, = struct.unpack('>I', tagx[4:8])
+        self.tagx_control_byte_count, = struct.unpack('>I', tagx[8:12])
+        tag_table = tagx[12:self.tagx_header_length]
+        if len(tag_table) % 4 != 0:
+            raise ValueError('Invalid Tag table')
+        num_tagx_entries = len(tag_table) // 4
+        self.tagx_entries = []
+        for i in range(num_tagx_entries):
+            self.tagx_entries.append(TagX(tag_table[i*4:(i+1)*4],
+                self.tagx_control_byte_count))
+        if self.tagx_entries and not self.tagx_entries[-1].is_eof:
+            raise ValueError('TAGX last entry is not EOF')
+        self.tagx_entries = self.tagx_entries[:-1]
+
+        idxt0_pos = self.header_length+self.tagx_header_length
+        last_num, consumed = decode_hex_number(raw[idxt0_pos:])
+        count_pos = idxt0_pos + consumed
+        self.ncx_count, = struct.unpack(b'>H', raw[count_pos:count_pos+2])
+
+        if last_num != self.ncx_count - 1:
+            raise ValueError('Last id number in the NCX != NCX count - 1')
+        # There may be some alignment zero bytes between the end of the idxt0
+        # and self.idxt_start
+
+        idxt = raw[self.idxt_start:]
+        if idxt[:4] != b'IDXT':
+            raise ValueError('Invalid IDXT header')
+        length_check, = struct.unpack(b'>H', idxt[4:6])
+        if length_check != self.header_length + self.tagx_header_length:
+            raise ValueError('Length check failed')
+
+    def __str__(self):
+        ans = ['*'*20 + ' Index Header '+ '*'*20]
+        a = ans.append
+        def u(w):
+            a('Unknown: %r (%d bytes) (All zeros: %r)'%(w,
+                len(w), not bool(w.replace(b'\0', b'')) ))
+
+        a('Header length: %d'%self.header_length)
+        u(self.unknown1)
+        a('Index Type: %s (%d)'%(self.index_type_desc, self.index_type))
+        a('Offset to IDXT start: %d'%self.idxt_start)
+        a('Number of index records: %d'%self.index_count)
+        a('Index encoding: %s (%d)'%(self.index_encoding,
+                self.index_encoding_num))
+        a('Index language: %s - %s (%s)'%(self.language, self.sublanguage,
+            hex(self.locale_raw)))
+        a('Number of index entries: %d'% self.num_index_entries)
+        a('ORDT start: %d'%self.ordt_start)
+        a('LIGT start: %d'%self.ligt_start)
+        a('Number of LIGT entries: %d'%self.num_of_ligt_entries)
+        a('Number of cncx blocks: %d'%self.num_of_cncx_blocks)
+        u(self.unknown2)
+        a('TAGX offset: %d'%self.tagx_offset)
+        u(self.unknown3)
+        a('\n\n')
+        a('*'*20 + ' TAGX Header (%d bytes)'%self.tagx_header_length+ '*'*20)
+        a('Header length: %d'%self.tagx_header_length)
+        a('Control byte count: %d'%self.tagx_control_byte_count)
+        for i in self.tagx_entries:
+            a('\t' + repr(i))
+        a('Number of entries in the NCX: %d'% self.ncx_count)
+
+        return '\n'.join(ans)
+    # }}}
+
+class Tag(object): # {{{
+
+    '''
+    Index entries are a collection of tags. Each tag is represented by this
+    class.
+    '''
+
+    TAG_MAP = {
+            1: ('offset', 'Offset in HTML'),
+            2: ('size', 'Size in HTML'),
+            3: ('label_offset', 'Offset to label in CNCX'),
+            4: ('depth', 'Depth of this entry in TOC'),
+
+            # The remaining tag types have to be interpreted subject to the type
+            # of index entry they are present in
+    }
+
+    INTERPRET_MAP = {
+            'subchapter': {
+                    5  : ('Parent chapter index', 'parent_index')
+            },
+
+            'article'   : {
+                    5  : ('Class offset in cncx', 'class_offset'),
+                    21 : ('Parent section index', 'parent_index'),
+                    22 : ('Description offset in cncx', 'desc_offset'),
+                    23 : ('Author offset in cncx', 'author_offset'),
+            },
+
+            'chapter_with_subchapters' : {
+                    22 : ('First subchapter index', 'first_subchapter_index'),
+                    23 : ('Last subchapter index', 'last_subchapter_index'),
+            },
+
+            'periodical' : {
+                    5  : ('Class offset in cncx', 'class_offset'),
+                    22 : ('First section index', 'first_section_index'),
+                    23 : ('Last section index', 'last_section_index'),
+            },
+
+            'section' : {
+                    5  : ('Class offset in cncx', 'class_offset'),
+                    21 : ('Periodical index', 'periodical_index'),
+                    22 : ('First article index', 'first_article_index'),
+                    23 : ('Last article index', 'last_article_index'),
+            },
+    }
+
+
+    def __init__(self, tagx, vals, entry_type, cncx):
+        self.value = vals if len(vals) > 1 else vals[0]
+        self.entry_type = entry_type
+        self.cncx_value = None
+        if tagx.tag in self.TAG_MAP:
+            self.attr, self.desc = self.TAG_MAP[tagx.tag]
+        else:
+            try:
+                td = self.INTERPRET_MAP[entry_type]
+            except:
+                raise ValueError('Unknown entry type: %s'%entry_type)
+            try:
+                self.desc, self.attr = td[tagx.tag]
+            except:
+                raise ValueError('Unknown tag: %d for entry type: %s'%(
+                    tagx.tag, entry_type))
+        if '_offset' in self.attr:
+            self.cncx_value = cncx[self.value]
+
+    def __str__(self):
+        if self.cncx_value is not None:
+            return '%s : %r [%r]'%(self.desc, self.value, self.cncx_value)
+        return '%s : %r'%(self.desc, self.value)
+
+# }}}
+
+class IndexEntry(object): # {{{
+
+    TYPES = {
+            # Present in book type files
+            0x0f : 'chapter',
+            0x6f : 'chapter_with_subchapters',
+            0x1f : 'subchapter',
+            # Present in periodicals
+            0xdf : 'periodical',
+            0xff : 'section',
+            0x3f : 'article',
+    }
+
+    def __init__(self, ident, entry_type, raw, cncx, tagx_entries):
+        self.index = ident
+        self.raw = raw
+        self.tags = []
+
+        try:
+            self.entry_type = self.TYPES[entry_type]
+        except KeyError:
+            raise ValueError('Unknown Index Entry type: %s'%hex(entry_type))
+
+        expected_tags = [tag for tag in tagx_entries if tag.bitmask &
+                entry_type]
+
+        for tag in expected_tags:
+            vals = []
+            for i in range(tag.num_values):
+                if not raw:
+                    raise ValueError('Index entry does not match TAGX header')
+                val, consumed = decint(raw)
+                raw = raw[consumed:]
+                vals.append(val)
+            self.tags.append(Tag(tag, vals, self.entry_type, cncx))
+
+    def __str__(self):
+        ans = ['Index Entry(index=%s, entry_type=%s, length=%d)'%(
+            self.index, self.entry_type, len(self.tags))]
+        for tag in self.tags:
+            ans.append('\t'+str(tag))
+        return '\n'.join(ans)
+
+# }}}
+
+class IndexRecord(object): # {{{
+
+    def __init__(self, record, index_header, cncx):
+        self.record = record
+        raw = self.record.raw
+        if raw[:4] != b'INDX':
+            raise ValueError('Invalid Primary Index Record')
+
+        u = struct.unpack
+
+        self.header_length, = u('>I', raw[4:8])
+        self.unknown1 = raw[8:12]
+        self.header_type, = u('>I', raw[12:16])
+        self.unknown2 = raw[16:20]
+        self.idxt_offset, self.idxt_count = u(b'>II', raw[20:28])
+        if self.idxt_offset < 192:
+            raise ValueError('Unknown Index record structure')
+        self.unknown3 = raw[28:36]
+        self.unknown4 = raw[36:192] # Should be 156 bytes
+
+        self.index_offsets = []
+        indices = raw[self.idxt_offset:]
+        if indices[:4] != b'IDXT':
+            raise ValueError("Invalid IDXT index table")
+        indices = indices[4:]
+        for i in range(self.idxt_count):
+            off, = u(b'>H', indices[i*2:(i+1)*2])
+            self.index_offsets.append(off-192)
+
+        indxt = raw[192:self.idxt_offset]
+        self.indices = []
+        for i, off in enumerate(self.index_offsets):
+            try:
+                next_off = self.index_offsets[i+1]
+            except:
+                next_off = len(indxt)
+            index, consumed = decode_hex_number(indxt[off:])
+            entry_type = ord(indxt[off+consumed])
+            self.indices.append(IndexEntry(index, entry_type,
+                indxt[off+consumed+1:next_off], cncx, index_header.tagx_entries))
+
+
+    def __str__(self):
+        ans = ['*'*20 + ' Index Record (%d bytes) '%len(self.record.raw)+ '*'*20]
+        a = ans.append
+        def u(w):
+            a('Unknown: %r (%d bytes) (All zeros: %r)'%(w,
+                len(w), not bool(w.replace(b'\0', b'')) ))
+        a('Header length: %d'%self.header_length)
+        u(self.unknown1)
+        a('Header Type: %d'%self.header_type)
+        u(self.unknown2)
+        a('IDXT Offset: %d'%self.idxt_offset)
+        a('IDXT Count: %d'%self.idxt_count)
+        u(self.unknown3)
+        u(self.unknown4)
+        a('Index offsets: %r'%self.index_offsets)
+        a('\nIndex Entries:')
+        for entry in self.indices:
+            a(str(entry)+'\n')
+
+        return '\n'.join(ans)
+
+# }}}
+
+class CNCX(object) : # {{{
+
+    def __init__(self, records, codec):
+        self.records = OrderedDict()
+        pos = 0
+        for record in records:
+            raw = record.raw
+            while pos < len(raw):
+                length, consumed = decint(raw[pos:])
+                if length > 0:
+                    self.records[pos] = raw[pos+consumed:pos+consumed+length].decode(
+                        codec)
+                pos += consumed+length
+
+    def __getitem__(self, offset):
+        return self.records.get(offset)
+
+    def __str__(self):
+        ans = ['*'*20 + ' cncx (%d strings) '%len(self.records)+ '*'*20]
+        for k, v in self.records.iteritems():
+            ans.append('%10d : %s'%(k, v))
+        return '\n'.join(ans)
+
+
+# }}}
+
+class MOBIFile(object): # {{{
 
     def __init__(self, stream):
         self.raw = stream.read()
@@ -384,25 +736,50 @@ class MOBIFile(object):
 
         self.mobi_header = MOBIHeader(self.records[0])
 
+        self.index_header = None
+        pir = self.mobi_header.primary_index_record
+        if pir != 0xffffffff:
+            self.index_header = IndexHeader(self.records[pir])
+            self.cncx = CNCX(self.records[
+                pir+2:pir+2+self.index_header.num_of_cncx_blocks],
+                self.index_header.index_encoding)
+            self.index_record = IndexRecord(self.records[pir+1],
+                    self.index_header, self.cncx)
 
-    def print_header(self):
-        print (str(self.palmdb).encode('utf-8'))
-        print ()
-        print ('Record headers:')
+
+    def print_header(self, f=sys.stdout):
+        print (str(self.palmdb).encode('utf-8'), file=f)
+        print (file=f)
+        print ('Record headers:', file=f)
         for i, r in enumerate(self.records):
-            print ('%6d. %s'%(i, r.header))
+            print ('%6d. %s'%(i, r.header), file=f)
 
-        print ()
-        print (str(self.mobi_header).encode('utf-8'))
+        print (file=f)
+        print (str(self.mobi_header).encode('utf-8'), file=f)
+# }}}
 
 def inspect_mobi(path_or_stream):
     stream = (path_or_stream if hasattr(path_or_stream, 'read') else
             open(path_or_stream, 'rb'))
     f = MOBIFile(stream)
-    f.print_header()
+    ddir = 'debug_' + os.path.splitext(os.path.basename(stream.name))[0]
+    if not os.path.exists(ddir):
+        os.mkdir(ddir)
+    with open(os.path.join(ddir, 'header.txt'), 'wb') as out:
+        f.print_header(f=out)
+    if f.index_header is not None:
+        with open(os.path.join(ddir, 'index.txt'), 'wb') as out:
+            print(str(f.index_header), file=out)
+            print('\n\n', file=out)
+            print(str(f.cncx).encode('utf-8'), file=out)
+            print('\n\n', file=out)
+            print(str(f.index_record), file=out)
+
+    print ('Debug data saved to:', ddir)
+
+def main():
+    inspect_mobi(sys.argv[1])
 
 if __name__ == '__main__':
-    import sys
-    f = MOBIFile(open(sys.argv[1], 'rb'))
-    f.print_header()
+    main()
 

src/calibre/ebooks/mobi/output.py

@@ -27,7 +27,7 @@ class MOBIOutput(OutputFormatPlugin):
         ),
         OptionRecommendation(name='no_inline_toc',
             recommended_value=False, level=OptionRecommendation.LOW,
-            help=_('Don\'t add Table of Contents to end of book. Useful if '
+            help=_('Don\'t add Table of Contents to the book. Useful if '
                 'the book has its own table of contents.')),
         OptionRecommendation(name='toc_title', recommended_value=None,
             help=_('Title for any generated in-line table of contents.')
@@ -45,6 +45,12 @@ class MOBIOutput(OutputFormatPlugin):
                 'the MOBI output plugin will try to convert margins specified'
                 ' in the input document, otherwise it will ignore them.')
         ),
+        OptionRecommendation(name='mobi_toc_at_start',
+            recommended_value=False,
+            help=_('When adding the Table of Contents to the book, add it at the start of the '
+                'book instead of the end. Not recommended.')
+        ),
+
     ])
 
     def check_for_periodical(self):
@@ -150,7 +156,7 @@ class MOBIOutput(OutputFormatPlugin):
             # Fix up the periodical href to point to first section href
             toc.nodes[0].href = toc.nodes[0].nodes[0].href
 
-            # GR diagnostics
+            # diagnostics
             if self.opts.verbose > 3:
                 self.dump_toc(toc)
                 self.dump_manifest()
@@ -158,16 +164,14 @@ class MOBIOutput(OutputFormatPlugin):
 
     def convert(self, oeb, output_path, input_plugin, opts, log):
         self.log, self.opts, self.oeb = log, opts, oeb
-        from calibre.ebooks.mobi.writer import PALM_MAX_IMAGE_SIZE, \
-                MobiWriter, PALMDOC, UNCOMPRESSED
         from calibre.ebooks.mobi.mobiml import MobiMLizer
         from calibre.ebooks.oeb.transforms.manglecase import CaseMangler
         from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable
         from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder
         from calibre.customize.ui import plugin_for_input_format
-        imagemax = PALM_MAX_IMAGE_SIZE if opts.rescale_images else None
         if not opts.no_inline_toc:
-            tocadder = HTMLTOCAdder(title=opts.toc_title)
+            tocadder = HTMLTOCAdder(title=opts.toc_title, position='start' if
+                    opts.mobi_toc_at_start else 'end')
             tocadder(oeb, opts)
         mangler = CaseMangler()
         mangler(oeb, opts)
@@ -179,10 +183,14 @@ class MOBIOutput(OutputFormatPlugin):
         mobimlizer = MobiMLizer(ignore_tables=opts.linearize_tables)
         mobimlizer(oeb, opts)
         self.check_for_periodical()
-        write_page_breaks_after_item = not input_plugin is plugin_for_input_format('cbz')
-        writer = MobiWriter(opts, imagemax=imagemax,
-                compression=UNCOMPRESSED if opts.dont_compress else PALMDOC,
-                            prefer_author_sort=opts.prefer_author_sort,
+        write_page_breaks_after_item = input_plugin is not plugin_for_input_format('cbz')
+        from calibre.utils.config import tweaks
+        if tweaks.get('new_mobi_writer', False):
+            from calibre.ebooks.mobi.writer2.main import MobiWriter
+            MobiWriter
+        else:
+            from calibre.ebooks.mobi.writer import MobiWriter
+        writer = MobiWriter(opts,
                         write_page_breaks_after_item=write_page_breaks_after_item)
         writer(oeb, output_path)
 

src/calibre/ebooks/mobi/writer.py

@@ -111,7 +111,8 @@ def align_block(raw, multiple=4, pad='\0'):
 
 def rescale_image(data, maxsizeb, dimen=None):
     if dimen is not None:
-        data = thumbnail(data, width=dimen, height=dimen)[-1]
+        data = thumbnail(data, width=dimen[0], height=dimen[1],
+                compression_quality=90)[-1]
     else:
         # Replace transparent pixels with white pixels and convert to JPEG
         data = save_cover_data_to(data, 'img.jpg', return_data=True)
@@ -141,7 +142,7 @@ def rescale_image(data, maxsizeb, dimen=None):
         scale -= 0.05
     return data
 
-class Serializer(object):
+class Serializer(object): # {{{
     NSRMAP = {'': None, XML_NS: 'xml', XHTML_NS: '', MBP_NS: 'mbp'}
 
     def __init__(self, oeb, images, write_page_breaks_after_item=True):
@@ -172,6 +173,9 @@ class Serializer(object):
         hrefs = self.oeb.manifest.hrefs
         buffer.write('<guide>')
         for ref in self.oeb.guide.values():
+            # The Kindle decides where to open a book based on the presence of
+            # an item in the guide that looks like
+            # <reference type="text" title="Start" href="chapter-one.xhtml"/>
             path = urldefrag(ref.href)[0]
             if path not in hrefs or hrefs[path].media_type not in OEB_DOCS:
                 continue
@@ -215,12 +219,6 @@ class Serializer(object):
         self.anchor_offset = buffer.tell()
         buffer.write('<body>')
         self.anchor_offset_kindle = buffer.tell()
-        # CybookG3 'Start Reading' link
-        if 'text' in self.oeb.guide:
-            href = self.oeb.guide['text'].href
-            buffer.write('<a ')
-            self.serialize_href(href)
-            buffer.write(' />')
         spine = [item for item in self.oeb.spine if item.linear]
         spine.extend([item for item in self.oeb.spine if not item.linear])
         for item in spine:
@@ -315,16 +313,20 @@ class Serializer(object):
                     buffer.seek(hoff)
                     buffer.write('%010d' % ioff)
 
+    # }}}
+
 class MobiWriter(object):
     COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+')
 
-    def __init__(self, opts, compression=PALMDOC, imagemax=None,
-            prefer_author_sort=False, write_page_breaks_after_item=True):
+    def __init__(self, opts,
+            write_page_breaks_after_item=True):
         self.opts = opts
         self.write_page_breaks_after_item = write_page_breaks_after_item
-        self._compression = compression or UNCOMPRESSED
-        self._imagemax = imagemax or OTHER_MAX_IMAGE_SIZE
-        self._prefer_author_sort = prefer_author_sort
+        self._compression = UNCOMPRESSED if getattr(opts, 'dont_compress',
+                False) else PALMDOC
+        self._imagemax = (PALM_MAX_IMAGE_SIZE if getattr(opts,
+            'rescale_images', False) else OTHER_MAX_IMAGE_SIZE)
+        self._prefer_author_sort = getattr(opts, 'prefer_author_sort', False)
         self._primary_index_record = None
         self._conforming_periodical_toc = False
         self._indexable = False
@@ -1258,11 +1260,11 @@ class MobiWriter(object):
                 data = compress_doc(data)
             record = StringIO()
             record.write(data)
-
-            # Marshall's utf-8 break code.
-            if WRITE_PBREAKS :
+            # Write trailing muti-byte sequence if any
             record.write(overlap)
             record.write(pack('>B', len(overlap)))
+
+            if WRITE_PBREAKS :
                 nextra = 0
                 pbreak = 0
                 running = offset
@@ -1325,6 +1327,8 @@ class MobiWriter(object):
             except:
                 self._oeb.logger.warn('Bad image file %r' % item.href)
                 continue
+            finally:
+                item.unload_data_from_memory()
             self._records.append(data)
             if self._first_image_record is None:
                 self._first_image_record = len(self._records)-1
@@ -1638,6 +1642,61 @@ class MobiWriter(object):
         for record in self._records:
             self._write(record)
 
+    def _clean_text_value(self, text):
+        if text is not None and text.strip() :
+            text = text.strip()
+            if not isinstance(text, unicode):
+                text = text.decode('utf-8', 'replace')
+            text = normalize(text).encode('utf-8')
+        else :
+            text = "(none)".encode('utf-8')
+        return text
+
+    def _compute_offset_length(self, i, node, entries) :
+        h = node.href
+        if h not in self._id_offsets:
+            self._oeb.log.warning('Could not find TOC entry:', node.title)
+            return -1, -1
+
+        offset = self._id_offsets[h]
+        length = None
+        # Calculate length based on next entry's offset
+        for sibling in entries[i+1:]:
+            h2 = sibling.href
+            if h2 in self._id_offsets:
+                offset2 = self._id_offsets[h2]
+                if offset2 > offset:
+                    length = offset2 - offset
+                    break
+        if length is None:
+            length = self._content_length - offset
+        return offset, length
+
+    def _establish_document_structure(self) :
+        documentType = None
+        try :
+            klass = self._ctoc_map[0]['klass']
+        except :
+            klass = None
+
+        if klass == 'chapter' or klass == None :
+            documentType = 'book'
+            if self.opts.verbose > 2 :
+                self._oeb.logger.info("Adding a MobiBook to self._MobiDoc")
+            self._MobiDoc.documentStructure = MobiBook()
+
+        elif klass == 'periodical' :
+            documentType = klass
+            if self.opts.verbose > 2 :
+                self._oeb.logger.info("Adding a MobiPeriodical to self._MobiDoc")
+            self._MobiDoc.documentStructure = MobiPeriodical(self._MobiDoc.getNextNode())
+            self._MobiDoc.documentStructure.startAddress = self._anchor_offset_kindle
+        else :
+            raise NotImplementedError('_establish_document_structure: unrecognized klass: %s' % klass)
+        return documentType
+
+    # Index {{{
+
     def _generate_index(self):
         self._oeb.log('Generating INDX ...')
         self._primary_index_record = None
@@ -1811,276 +1870,7 @@ class MobiWriter(object):
                     open(os.path.join(t, n+'.bin'), 'wb').write(self._records[-(i+1)])
                 self._oeb.log.debug('Index records dumped to', t)
 
-    def _clean_text_value(self, text):
-        if text is not None and text.strip() :
-            text = text.strip()
-            if not isinstance(text, unicode):
-                text = text.decode('utf-8', 'replace')
-            text = normalize(text).encode('utf-8')
-        else :
-            text = "(none)".encode('utf-8')
-        return text
-
-    def _add_to_ctoc(self, ctoc_str, record_offset):
-        # Write vwilen + string to ctoc
-        # Return offset
-        # Is there enough room for this string in the current ctoc record?
-        if 0xfbf8 - self._ctoc.tell() < 2 + len(ctoc_str):
-            # flush this ctoc, start a new one
-            # print "closing ctoc_record at 0x%X" % self._ctoc.tell()
-            # print "starting new ctoc with '%-50.50s ...'" % ctoc_str
-            # pad with 00
-            pad = 0xfbf8 - self._ctoc.tell()
-            # print "padding %d bytes of 00" % pad
-            self._ctoc.write('\0' * (pad))
-            self._ctoc_records.append(self._ctoc.getvalue())
-            self._ctoc.truncate(0)
-            self._ctoc_offset += 0x10000
-            record_offset = self._ctoc_offset
-
-        offset = self._ctoc.tell() + record_offset
-        self._ctoc.write(decint(len(ctoc_str), DECINT_FORWARD) + ctoc_str)
-        return offset
-
-    def _add_flat_ctoc_node(self, node, ctoc, title=None):
-        # Process 'chapter' or 'article' nodes only, force either to 'chapter'
-        t = node.title if title is None else title
-        t = self._clean_text_value(t)
-        self._last_toc_entry = t
-
-        # Create an empty dictionary for this node
-        ctoc_name_map = {}
-
-        # article = chapter
-        if node.klass == 'article' :
-            ctoc_name_map['klass'] = 'chapter'
-        else :
-            ctoc_name_map['klass'] = node.klass
-
-        # Add title offset to name map
-        ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
-        self._chapterCount += 1
-
-        # append this node's name_map to map
-        self._ctoc_map.append(ctoc_name_map)
-
-        return
-
-    def _add_structured_ctoc_node(self, node, ctoc, title=None):
-        # Process 'periodical', 'section' and 'article'
-
-        # Fetch the offset referencing the current ctoc_record
-        if node.klass is None :
-            return
-        t = node.title if title is None else title
-        t = self._clean_text_value(t)
-        self._last_toc_entry = t
-
-        # Create an empty dictionary for this node
-        ctoc_name_map = {}
-
-        # Add the klass of this node
-        ctoc_name_map['klass'] = node.klass
-
-        if node.klass == 'chapter':
-            # Add title offset to name map
-            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
-            self._chapterCount += 1
-
-        elif node.klass == 'periodical' :
-            # Add title offset
-            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
-
-            # Look for existing class entry 'periodical' in _ctoc_map
-            for entry in self._ctoc_map:
-                if entry['klass'] == 'periodical':
-                    # Use the pre-existing instance
-                    ctoc_name_map['classOffset'] = entry['classOffset']
-                    break
-                else :
-                    continue
-            else:
-                # class names should always be in CNCX 0 - no offset
-                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
-
-            self._periodicalCount += 1
-
-        elif node.klass == 'section' :
-            # Add title offset
-            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
-
-            # Look for existing class entry 'section' in _ctoc_map
-            for entry in self._ctoc_map:
-                if entry['klass'] == 'section':
-                    # Use the pre-existing instance
-                    ctoc_name_map['classOffset'] = entry['classOffset']
-                    break
-                else :
-                    continue
-            else:
-                # class names should always be in CNCX 0 - no offset
-                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
-
-            self._sectionCount += 1
-
-        elif node.klass == 'article' :
-            # Add title offset/title
-            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
-
-            # Look for existing class entry 'article' in _ctoc_map
-            for entry in self._ctoc_map:
-                if entry['klass'] == 'article':
-                    ctoc_name_map['classOffset'] = entry['classOffset']
-                    break
-                else :
-                    continue
-            else:
-                # class names should always be in CNCX 0 - no offset
-                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
-
-            # Add description offset/description
-            if node.description :
-                d = self._clean_text_value(node.description)
-                ctoc_name_map['descriptionOffset'] = self._add_to_ctoc(d, self._ctoc_offset)
-            else :
-                ctoc_name_map['descriptionOffset'] = None
-
-            # Add author offset/attribution
-            if node.author :
-                a = self._clean_text_value(node.author)
-                ctoc_name_map['authorOffset'] = self._add_to_ctoc(a, self._ctoc_offset)
-            else :
-                ctoc_name_map['authorOffset'] = None
-
-            self._articleCount += 1
-
-        else :
-            raise NotImplementedError( \
-            'writer._generate_ctoc.add_node: title: %s has unrecognized klass: %s, playOrder: %d' % \
-            (node.title, node.klass, node.play_order))
-
-        # append this node's name_map to map
-        self._ctoc_map.append(ctoc_name_map)
-
-    def _generate_ctoc(self):
-        # Generate the compiled TOC strings
-        # Each node has 1-4 CTOC entries:
-        #	Periodical (0xDF)
-        #		title, class
-        #	Section (0xFF)
-        #		title, class
-        #	Article (0x3F)
-        #		title, class, description, author
-        #	Chapter (0x0F)
-        #		title, class
-        #   nb: Chapters don't actually have @class, so we synthesize it
-        #   in reader._toc_from_navpoint
-
-        toc = self._oeb.toc
-        reduced_toc = []
-        self._ctoc_map = []				# per node dictionary of {class/title/desc/author} offsets
-        self._last_toc_entry = None
-        #ctoc = StringIO()
-        self._ctoc = StringIO()
-
-        # Track the individual node types
-        self._periodicalCount = 0
-        self._sectionCount = 0
-        self._articleCount = 0
-        self._chapterCount = 0
-
-        #first = True
-
-        if self._conforming_periodical_toc :
-            self._oeb.logger.info('Generating structured CTOC ...')
-            for (child) in toc.iter():
-                if self.opts.verbose > 2 :
-                    self._oeb.logger.info("  %s" % child)
-                self._add_structured_ctoc_node(child, self._ctoc)
-                #first = False
-
-        else :
-            self._oeb.logger.info('Generating flat CTOC ...')
-            previousOffset = -1
-            currentOffset = 0
-            for (i, child) in enumerate(toc.iterdescendants()):
-                # Only add chapters or articles at depth==1
-                # no class defaults to 'chapter'
-                if child.klass is None : child.klass = 'chapter'
-                if (child.klass == 'article' or child.klass == 'chapter') and child.depth() == 1 :
-                    if self.opts.verbose > 2 :
-                        self._oeb.logger.info("adding (klass:%s depth:%d) %s to flat ctoc" % \
-                                              (child.klass, child.depth(), child) )
-
-                    # Test to see if this child's offset is the same as the previous child's
-                    # offset, skip it
-                    h = child.href
-
-                    if h is None:
-                        self._oeb.logger.warn('  Ignoring TOC entry with no href:',
-                                child.title)
-                        continue
-                    if h not in self._id_offsets:
-                        self._oeb.logger.warn('  Ignoring missing TOC entry:',
-                                unicode(child))
-                        continue
-
-                    currentOffset = self._id_offsets[h]
-                    # print "_generate_ctoc: child offset: 0x%X" % currentOffset
-
-                    if currentOffset != previousOffset :
-                        self._add_flat_ctoc_node(child, self._ctoc)
-                        reduced_toc.append(child)
-                        previousOffset = currentOffset
-                    else :
-                        self._oeb.logger.warn("  Ignoring redundant href: %s in '%s'" % (h, child.title))
-
-                else :
-                    if self.opts.verbose > 2 :
-                        self._oeb.logger.info("skipping class: %s depth %d at position %d" % \
-                                              (child.klass, child.depth(),i))
-
-            # Update the TOC with our edited version
-            self._oeb.toc.nodes = reduced_toc
-
-        # Instantiate a MobiDocument(mobitype)
-        if (not self._periodicalCount and not self._sectionCount and not self._articleCount) or \
-            not self.opts.mobi_periodical :
-            mobiType = 0x002
-        elif self._periodicalCount:
-            pt = None
-            if self._oeb.metadata.publication_type:
-                x = unicode(self._oeb.metadata.publication_type[0]).split(':')
-                if len(x) > 1:
-                    pt = x[1]
-            mobiType = {'newspaper':0x101}.get(pt, 0x103)
-        else :
-            raise NotImplementedError('_generate_ctoc: Unrecognized document structured')
-
-        self._MobiDoc = MobiDocument(mobiType)
-
-        if self.opts.verbose > 2 :
-            structType = 'book'
-            if mobiType > 0x100 :
-                structType = 'flat periodical' if mobiType == 0x102 else 'structured periodical'
-            self._oeb.logger.info("Instantiating a %s MobiDocument of type 0x%X" % (structType, mobiType ) )
-            if mobiType > 0x100 :
-                self._oeb.logger.info("periodicalCount: %d  sectionCount: %d  articleCount: %d"% \
-                                    (self._periodicalCount, self._sectionCount, self._articleCount) )
-            else :
-                self._oeb.logger.info("chapterCount: %d" % self._chapterCount)
-
-        # Apparently the CTOC must end with a null byte
-        self._ctoc.write('\0')
-
-        ctoc = self._ctoc.getvalue()
-        rec_count = len(self._ctoc_records)
-        self._oeb.logger.info("  CNCX utilization: %d %s %.0f%% full" % \
-            (rec_count + 1, 'records, last record' if rec_count else 'record,',
-                len(ctoc)/655) )
-
-        return align_block(ctoc)
-
+    # Index nodes {{{
     def _write_periodical_node(self, indxt, indices, index, offset, length, count, firstSection, lastSection) :
         pos = 0xc0 + indxt.tell()
         indices.write(pack('>H', pos))								# Save the offset for IDXTIndices
@@ -2172,48 +1962,8 @@ class MobiWriter(object):
         indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD))	# vwi title offset in CNCX
         indxt.write(decint(0, DECINT_FORWARD))						# unknown byte
 
-    def _compute_offset_length(self, i, node, entries) :
-        h = node.href
-        if h not in self._id_offsets:
-            self._oeb.log.warning('Could not find TOC entry:', node.title)
-            return -1, -1
+    # }}}
 
-        offset = self._id_offsets[h]
-        length = None
-        # Calculate length based on next entry's offset
-        for sibling in entries[i+1:]:
-            h2 = sibling.href
-            if h2 in self._id_offsets:
-                offset2 = self._id_offsets[h2]
-                if offset2 > offset:
-                    length = offset2 - offset
-                    break
-        if length is None:
-            length = self._content_length - offset
-        return offset, length
-
-    def _establish_document_structure(self) :
-        documentType = None
-        try :
-            klass = self._ctoc_map[0]['klass']
-        except :
-            klass = None
-
-        if klass == 'chapter' or klass == None :
-            documentType = 'book'
-            if self.opts.verbose > 2 :
-                self._oeb.logger.info("Adding a MobiBook to self._MobiDoc")
-            self._MobiDoc.documentStructure = MobiBook()
-
-        elif klass == 'periodical' :
-            documentType = klass
-            if self.opts.verbose > 2 :
-                self._oeb.logger.info("Adding a MobiPeriodical to self._MobiDoc")
-            self._MobiDoc.documentStructure = MobiPeriodical(self._MobiDoc.getNextNode())
-            self._MobiDoc.documentStructure.startAddress = self._anchor_offset_kindle
-        else :
-            raise NotImplementedError('_establish_document_structure: unrecognized klass: %s' % klass)
-        return documentType
 
     def _generate_section_indices(self, child, currentSection, myPeriodical, myDoc ) :
         sectionTitles = list(child.iter())[1:]
@@ -2491,6 +2241,270 @@ class MobiWriter(object):
             last_name, c = self._add_periodical_structured_articles(myDoc, indxt, indices)
 
         return align_block(indxt.getvalue()), c, align_block(indices.getvalue()), last_name
+    # }}}
+
+    # CTOC {{{
+    def _add_to_ctoc(self, ctoc_str, record_offset):
+        # Write vwilen + string to ctoc
+        # Return offset
+        # Is there enough room for this string in the current ctoc record?
+        if 0xfbf8 - self._ctoc.tell() < 2 + len(ctoc_str):
+            # flush this ctoc, start a new one
+            # print "closing ctoc_record at 0x%X" % self._ctoc.tell()
+            # print "starting new ctoc with '%-50.50s ...'" % ctoc_str
+            # pad with 00
+            pad = 0xfbf8 - self._ctoc.tell()
+            # print "padding %d bytes of 00" % pad
+            self._ctoc.write('\0' * (pad))
+            self._ctoc_records.append(self._ctoc.getvalue())
+            self._ctoc.truncate(0)
+            self._ctoc_offset += 0x10000
+            record_offset = self._ctoc_offset
+
+        offset = self._ctoc.tell() + record_offset
+        self._ctoc.write(decint(len(ctoc_str), DECINT_FORWARD) + ctoc_str)
+        return offset
+
+    def _add_flat_ctoc_node(self, node, ctoc, title=None):
+        # Process 'chapter' or 'article' nodes only, force either to 'chapter'
+        t = node.title if title is None else title
+        t = self._clean_text_value(t)
+        self._last_toc_entry = t
+
+        # Create an empty dictionary for this node
+        ctoc_name_map = {}
+
+        # article = chapter
+        if node.klass == 'article' :
+            ctoc_name_map['klass'] = 'chapter'
+        else :
+            ctoc_name_map['klass'] = node.klass
+
+        # Add title offset to name map
+        ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
+        self._chapterCount += 1
+
+        # append this node's name_map to map
+        self._ctoc_map.append(ctoc_name_map)
+
+        return
+
+    def _add_structured_ctoc_node(self, node, ctoc, title=None):
+        # Process 'periodical', 'section' and 'article'
+
+        # Fetch the offset referencing the current ctoc_record
+        if node.klass is None :
+            return
+        t = node.title if title is None else title
+        t = self._clean_text_value(t)
+        self._last_toc_entry = t
+
+        # Create an empty dictionary for this node
+        ctoc_name_map = {}
+
+        # Add the klass of this node
+        ctoc_name_map['klass'] = node.klass
+
+        if node.klass == 'chapter':
+            # Add title offset to name map
+            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
+            self._chapterCount += 1
+
+        elif node.klass == 'periodical' :
+            # Add title offset
+            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
+
+            # Look for existing class entry 'periodical' in _ctoc_map
+            for entry in self._ctoc_map:
+                if entry['klass'] == 'periodical':
+                    # Use the pre-existing instance
+                    ctoc_name_map['classOffset'] = entry['classOffset']
+                    break
+                else :
+                    continue
+            else:
+                # class names should always be in CNCX 0 - no offset
+                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
+
+            self._periodicalCount += 1
+
+        elif node.klass == 'section' :
+            # Add title offset
+            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
+
+            # Look for existing class entry 'section' in _ctoc_map
+            for entry in self._ctoc_map:
+                if entry['klass'] == 'section':
+                    # Use the pre-existing instance
+                    ctoc_name_map['classOffset'] = entry['classOffset']
+                    break
+                else :
+                    continue
+            else:
+                # class names should always be in CNCX 0 - no offset
+                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
+
+            self._sectionCount += 1
+
+        elif node.klass == 'article' :
+            # Add title offset/title
+            ctoc_name_map['titleOffset'] = self._add_to_ctoc(t, self._ctoc_offset)
+
+            # Look for existing class entry 'article' in _ctoc_map
+            for entry in self._ctoc_map:
+                if entry['klass'] == 'article':
+                    ctoc_name_map['classOffset'] = entry['classOffset']
+                    break
+                else :
+                    continue
+            else:
+                # class names should always be in CNCX 0 - no offset
+                ctoc_name_map['classOffset'] = self._add_to_ctoc(node.klass, 0)
+
+            # Add description offset/description
+            if node.description :
+                d = self._clean_text_value(node.description)
+                ctoc_name_map['descriptionOffset'] = self._add_to_ctoc(d, self._ctoc_offset)
+            else :
+                ctoc_name_map['descriptionOffset'] = None
+
+            # Add author offset/attribution
+            if node.author :
+                a = self._clean_text_value(node.author)
+                ctoc_name_map['authorOffset'] = self._add_to_ctoc(a, self._ctoc_offset)
+            else :
+                ctoc_name_map['authorOffset'] = None
+
+            self._articleCount += 1
+
+        else :
+            raise NotImplementedError( \
+            'writer._generate_ctoc.add_node: title: %s has unrecognized klass: %s, playOrder: %d' % \
+            (node.title, node.klass, node.play_order))
+
+        # append this node's name_map to map
+        self._ctoc_map.append(ctoc_name_map)
+
+    def _generate_ctoc(self):
+        # Generate the compiled TOC strings
+        # Each node has 1-4 CTOC entries:
+        #	Periodical (0xDF)
+        #		title, class
+        #	Section (0xFF)
+        #		title, class
+        #	Article (0x3F)
+        #		title, class, description, author
+        #	Chapter (0x0F)
+        #		title, class
+        #   nb: Chapters don't actually have @class, so we synthesize it
+        #   in reader._toc_from_navpoint
+
+        toc = self._oeb.toc
+        reduced_toc = []
+        self._ctoc_map = []				# per node dictionary of {class/title/desc/author} offsets
+        self._last_toc_entry = None
+        #ctoc = StringIO()
+        self._ctoc = StringIO()
+
+        # Track the individual node types
+        self._periodicalCount = 0
+        self._sectionCount = 0
+        self._articleCount = 0
+        self._chapterCount = 0
+
+        #first = True
+
+        if self._conforming_periodical_toc :
+            self._oeb.logger.info('Generating structured CTOC ...')
+            for (child) in toc.iter():
+                if self.opts.verbose > 2 :
+                    self._oeb.logger.info("  %s" % child)
+                self._add_structured_ctoc_node(child, self._ctoc)
+                #first = False
+
+        else :
+            self._oeb.logger.info('Generating flat CTOC ...')
+            previousOffset = -1
+            currentOffset = 0
+            for (i, child) in enumerate(toc.iterdescendants()):
+                # Only add chapters or articles at depth==1
+                # no class defaults to 'chapter'
+                if child.klass is None : child.klass = 'chapter'
+                if (child.klass == 'article' or child.klass == 'chapter') and child.depth() == 1 :
+                    if self.opts.verbose > 2 :
+                        self._oeb.logger.info("adding (klass:%s depth:%d) %s to flat ctoc" % \
+                                              (child.klass, child.depth(), child) )
+
+                    # Test to see if this child's offset is the same as the previous child's
+                    # offset, skip it
+                    h = child.href
+
+                    if h is None:
+                        self._oeb.logger.warn('  Ignoring TOC entry with no href:',
+                                child.title)
+                        continue
+                    if h not in self._id_offsets:
+                        self._oeb.logger.warn('  Ignoring missing TOC entry:',
+                                unicode(child))
+                        continue
+
+                    currentOffset = self._id_offsets[h]
+                    # print "_generate_ctoc: child offset: 0x%X" % currentOffset
+
+                    if currentOffset != previousOffset :
+                        self._add_flat_ctoc_node(child, self._ctoc)
+                        reduced_toc.append(child)
+                        previousOffset = currentOffset
+                    else :
+                        self._oeb.logger.warn("  Ignoring redundant href: %s in '%s'" % (h, child.title))
+
+                else :
+                    if self.opts.verbose > 2 :
+                        self._oeb.logger.info("skipping class: %s depth %d at position %d" % \
+                                              (child.klass, child.depth(),i))
+
+            # Update the TOC with our edited version
+            self._oeb.toc.nodes = reduced_toc
+
+        # Instantiate a MobiDocument(mobitype)
+        if (not self._periodicalCount and not self._sectionCount and not self._articleCount) or \
+            not self.opts.mobi_periodical :
+            mobiType = 0x002
+        elif self._periodicalCount:
+            pt = None
+            if self._oeb.metadata.publication_type:
+                x = unicode(self._oeb.metadata.publication_type[0]).split(':')
+                if len(x) > 1:
+                    pt = x[1]
+            mobiType = {'newspaper':0x101}.get(pt, 0x103)
+        else :
+            raise NotImplementedError('_generate_ctoc: Unrecognized document structured')
+
+        self._MobiDoc = MobiDocument(mobiType)
+
+        if self.opts.verbose > 2 :
+            structType = 'book'
+            if mobiType > 0x100 :
+                structType = 'flat periodical' if mobiType == 0x102 else 'structured periodical'
+            self._oeb.logger.info("Instantiating a %s MobiDocument of type 0x%X" % (structType, mobiType ) )
+            if mobiType > 0x100 :
+                self._oeb.logger.info("periodicalCount: %d  sectionCount: %d  articleCount: %d"% \
+                                    (self._periodicalCount, self._sectionCount, self._articleCount) )
+            else :
+                self._oeb.logger.info("chapterCount: %d" % self._chapterCount)
+
+        # Apparently the CTOC must end with a null byte
+        self._ctoc.write('\0')
+
+        ctoc = self._ctoc.getvalue()
+        rec_count = len(self._ctoc_records)
+        self._oeb.logger.info("  CNCX utilization: %d %s %.0f%% full" % \
+            (rec_count + 1, 'records, last record' if rec_count else 'record,',
+                len(ctoc)/655) )
+
+        return align_block(ctoc)
+
+    # }}}
 
 class HTMLRecordData(object):
     """ A data structure containing indexing/navigation data for an HTML record """

src/calibre/ebooks/mobi/writer2/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+UNCOMPRESSED = 1
+PALMDOC = 2
+HUFFDIC = 17480
+PALM_MAX_IMAGE_SIZE = 63 * 1024
+

src/calibre/ebooks/mobi/writer2/main.py

@@ -0,0 +1,502 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import re, random, time
+from cStringIO import StringIO
+from struct import pack
+
+from calibre.ebooks import normalize
+from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
+from calibre.ebooks.mobi.writer2.serializer import Serializer
+from calibre.ebooks.compression.palmdoc import compress_doc
+from calibre.ebooks.mobi.langcodes import iana2mobi
+from calibre.utils.filenames import ascii_filename
+from calibre.ebooks.mobi.writer2 import PALMDOC, UNCOMPRESSED
+from calibre.ebooks.mobi.writer2.utils import (rescale_image, encint)
+
+EXTH_CODES = {
+    'creator': 100,
+    'publisher': 101,
+    'description': 103,
+    'identifier': 104,
+    'subject': 105,
+    'pubdate': 106,
+    'date': 106,
+    'review': 107,
+    'contributor': 108,
+    'rights': 109,
+    'type': 111,
+    'source': 112,
+    'title': 503,
+    }
+
+# Disabled as I dont care about uncrossable breaks
+WRITE_UNCROSSABLE_BREAKS = False
+
+RECORD_SIZE = 0x1000 # 4096
+
+
+MAX_THUMB_SIZE = 16 * 1024
+MAX_THUMB_DIMEN = (180, 240)
+
+class MobiWriter(object):
+    COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+')
+
+    def __init__(self, opts, write_page_breaks_after_item=True):
+        self.opts = opts
+        self.write_page_breaks_after_item = write_page_breaks_after_item
+        self.compression = UNCOMPRESSED if opts.dont_compress else PALMDOC
+        self.prefer_author_sort = opts.prefer_author_sort
+
+    def __call__(self, oeb, path_or_stream):
+        if hasattr(path_or_stream, 'write'):
+            return self.dump_stream(oeb, path_or_stream)
+        with open(path_or_stream, 'w+b') as stream:
+            return self.dump_stream(oeb, stream)
+
+    def write(self, *args):
+        for datum in args:
+            self.stream.write(datum)
+
+    def tell(self):
+        return self.stream.tell()
+
+    def dump_stream(self, oeb, stream):
+        self.oeb = oeb
+        self.stream = stream
+        self.records = [None]
+        self.generate_content()
+        self.generate_record0()
+        self.write_header()
+        self.write_content()
+
+    def generate_content(self):
+        self.map_image_names()
+        self.generate_text()
+        # Image records come after text records
+        self.generate_images()
+
+    def map_image_names(self):
+        '''
+        Map image names to record indices, ensuring that the masthead image if
+        present has index number 1.
+        '''
+        index = 1
+        self.images = images = {}
+        mh_href = None
+
+        if 'masthead' in self.oeb.guide:
+            mh_href = self.oeb.guide['masthead'].href
+            images[mh_href] = 1
+            index += 1
+
+        for item in self.oeb.manifest.values():
+            if item.media_type in OEB_RASTER_IMAGES:
+                if item.href == mh_href: continue
+                images[item.href] = index
+                index += 1
+
+    def generate_images(self):
+        self.oeb.logger.info('Serializing images...')
+        images = [(index, href) for href, index in self.images.iteritems()]
+        images.sort()
+        self.first_image_record = None
+        for _, href in images:
+            item = self.oeb.manifest.hrefs[href]
+            try:
+                data = rescale_image(item.data)
+            except:
+                self.oeb.logger.warn('Bad image file %r' % item.href)
+                continue
+            finally:
+                item.unload_data_from_memory()
+            self.records.append(data)
+            if self.first_image_record is None:
+                self.first_image_record = len(self.records) - 1
+
+    def generate_text(self):
+        self.oeb.logger.info('Serializing markup content...')
+        serializer = Serializer(self.oeb, self.images,
+                write_page_breaks_after_item=self.write_page_breaks_after_item)
+        text = serializer()
+        breaks = serializer.breaks
+        self.anchor_offset_kindle = serializer.anchor_offset_kindle
+        self.id_offsets = serializer.id_offsets
+        self.content_length = len(text)
+        self.text_length = len(text)
+        text = StringIO(text)
+        buf = []
+        nrecords = 0
+        offset = 0
+
+        if self.compression != UNCOMPRESSED:
+            self.oeb.logger.info('  Compressing markup content...')
+        data, overlap = self.read_text_record(text)
+
+        while len(data) > 0:
+            if self.compression == PALMDOC:
+                data = compress_doc(data)
+            record = StringIO()
+            record.write(data)
+
+            self.records.append(record.getvalue())
+            buf.append(self.records[-1])
+            nrecords += 1
+            offset += RECORD_SIZE
+            data, overlap = self.read_text_record(text)
+
+            # Write information about the mutibyte character overlap, if any
+            record.write(overlap)
+            record.write(pack(b'>B', len(overlap)))
+
+            # Write information about uncrossable breaks (non linear items in
+            # the spine)
+            if WRITE_UNCROSSABLE_BREAKS:
+                nextra = 0
+                pbreak = 0
+                running = offset
+
+                # Write information about every uncrossable break that occurs in
+                # the next record.
+                while breaks and (breaks[0] - offset) < RECORD_SIZE:
+                    pbreak = (breaks.pop(0) - running) >> 3
+                    encoded = encint(pbreak)
+                    record.write(encoded)
+                    running += pbreak << 3
+                    nextra += len(encoded)
+                lsize = 1
+                while True:
+                    size = encint(nextra + lsize, forward=False)
+                    if len(size) == lsize:
+                        break
+                    lsize += 1
+                record.write(size)
+
+        self.text_nrecords = nrecords + 1
+
+    def read_text_record(self, text):
+        '''
+        Return a Palmdoc record of size RECORD_SIZE from the text file object.
+        In case the record ends in the middle of a multibyte character return
+        the overlap as well.
+
+        Returns data, overlap: where both are byte strings. overlap is the
+        extra bytes needed to complete the truncated multibyte character.
+        '''
+        opos = text.tell()
+        text.seek(0, 2)
+        # npos is the position of the next record
+        npos = min((opos + RECORD_SIZE, text.tell()))
+        # Number of bytes from the next record needed to complete the last
+        # character in this record
+        extra = 0
+
+        last = b''
+        while not last.decode('utf-8', 'ignore'):
+            # last contains no valid utf-8 characters
+            size = len(last) + 1
+            text.seek(npos - size)
+            last = text.read(size)
+
+        # last now has one valid utf-8 char and possibly some bytes that belong
+        # to a truncated char
+
+        try:
+            last.decode('utf-8', 'strict')
+        except UnicodeDecodeError:
+            # There are some truncated bytes in last
+            prev = len(last)
+            while True:
+                text.seek(npos - prev)
+                last = text.read(len(last) + 1)
+                try:
+                    last.decode('utf-8')
+                except UnicodeDecodeError:
+                    pass
+                else:
+                    break
+            extra = len(last) - prev
+
+        text.seek(opos)
+        data = text.read(RECORD_SIZE)
+        overlap = text.read(extra)
+        text.seek(npos)
+
+        return data, overlap
+
+    def generate_end_records(self):
+        self.flis_number = len(self.records)
+        self.records.append('\xE9\x8E\x0D\x0A')
+
+    def generate_record0(self): # {{{
+        metadata = self.oeb.metadata
+        exth = self.build_exth()
+        last_content_record = len(self.records) - 1
+
+        self.generate_end_records()
+
+        record0 = StringIO()
+        # The PalmDOC Header
+        record0.write(pack(b'>HHIHHHH', self.compression, 0,
+            self.text_length,
+            self.text_nrecords-1, RECORD_SIZE, 0, 0)) # 0 - 15 (0x0 - 0xf)
+        uid = random.randint(0, 0xffffffff)
+        title = normalize(unicode(metadata.title[0])).encode('utf-8')
+        # The MOBI Header
+
+        # 0x0 - 0x3
+        record0.write(b'MOBI')
+
+        # 0x4 - 0x7   : Length of header
+        # 0x8 - 0x11  : MOBI type
+        #   type    meaning
+        #   0x002   MOBI book (chapter - chapter navigation)
+        #   0x101   News - Hierarchical navigation with sections and articles
+        #   0x102   News feed - Flat navigation
+        #   0x103   News magazine - same as 0x101
+        # 0xC - 0xF   : Text encoding (65001 is utf-8)
+        # 0x10 - 0x13 : UID
+        # 0x14 - 0x17 : Generator version
+
+        record0.write(pack(b'>IIIII',
+            0xe8, 0x002, 65001, uid, 6))
+
+        # 0x18 - 0x1f : Unknown
+        record0.write(b'\xff' * 8)
+
+
+        # 0x20 - 0x23 : Secondary index record
+        record0.write(pack(b'>I', 0xffffffff))
+
+        # 0x24 - 0x3f : Unknown
+        record0.write(b'\xff' * 28)
+
+        # 0x40 - 0x43 : Offset of first non-text record
+        record0.write(pack(b'>I',
+            self.text_nrecords + 1))
+
+        # 0x44 - 0x4b : title offset, title length
+        record0.write(pack(b'>II',
+            0xe8 + 16 + len(exth), len(title)))
+
+        # 0x4c - 0x4f : Language specifier
+        record0.write(iana2mobi(
+            str(metadata.language[0])))
+
+        # 0x50 - 0x57 : Unknown
+        record0.write(b'\0' * 8)
+
+        # 0x58 - 0x5b : Format version
+        # 0x5c - 0x5f : First image record number
+        record0.write(pack(b'>II',
+            6, self.first_image_record if self.first_image_record else 0))
+
+        # 0x60 - 0x63 : First HUFF/CDIC record number
+        # 0x64 - 0x67 : Number of HUFF/CDIC records
+        # 0x68 - 0x6b : First DATP record number
+        # 0x6c - 0x6f : Number of DATP records
+        record0.write(b'\0' * 16)
+
+        # 0x70 - 0x73 : EXTH flags
+        record0.write(pack(b'>I', 0x50))
+
+        # 0x74 - 0x93 : Unknown
+        record0.write(b'\0' * 32)
+
+        # 0x94 - 0x97 : DRM offset
+        # 0x98 - 0x9b : DRM count
+        # 0x9c - 0x9f : DRM size
+        # 0xa0 - 0xa3 : DRM flags
+        record0.write(pack(b'>IIII',
+            0xffffffff, 0xffffffff, 0, 0))
+
+
+        # 0xa4 - 0xaf : Unknown
+        record0.write(b'\0'*12)
+
+        # 0xb0 - 0xb1 : First content record number
+        # 0xb2 - 0xb3 : last content record number
+        # (Includes Image, DATP, HUFF, DRM)
+        record0.write(pack(b'>HH', 1, last_content_record))
+
+        # 0xb4 - 0xb7 : Unknown
+        record0.write(b'\0\0\0\x01')
+
+        # 0xb8 - 0xbb : FCIS record number
+        record0.write(pack(b'>I', 0xffffffff))
+
+        # 0xbc - 0xbf : Unknown (FCIS record count?)
+        record0.write(pack(b'>I', 0xffffffff))
+
+        # 0xc0 - 0xc3 : FLIS record number
+        record0.write(pack(b'>I', 0xffffffff))
+
+        # 0xc4 - 0xc7 : Unknown (FLIS record count?)
+        record0.write(pack(b'>I', 1))
+
+        # 0xc8 - 0xcf : Unknown
+        record0.write(b'\0'*8)
+
+        # 0xd0 - 0xdf : Unknown
+        record0.write(pack(b'>IIII', 0xffffffff, 0, 0xffffffff, 0xffffffff))
+
+        # 0xe0 - 0xe3 : Extra record data
+        # Extra record data flags:
+        #   - 0x1: <extra multibyte bytes><size> (?)
+        #   - 0x2: <TBS indexing description of this HTML record><size> GR
+        #   - 0x4: <uncrossable breaks><size>
+        # GR: Use 7 for indexed files, 5 for unindexed
+        # Setting bit 2 (0x2) disables <guide><reference type="start"> functionality
+
+        extra_data_flags = 0b1 # Has multibyte overlap bytes
+        if WRITE_UNCROSSABLE_BREAKS:
+            extra_data_flags |= 0b100
+        record0.write(pack(b'>I', extra_data_flags))
+
+        # 0xe4 - 0xe7 : Primary index record
+        record0.write(pack(b'>I', 0xffffffff))
+
+        record0.write(exth)
+        record0.write(title)
+        record0 = record0.getvalue()
+        # Add some buffer so that Amazon can add encryption information if this
+        # MOBI is submitted for publication
+        record0 += (b'\0' * (1024*8))
+        self.records[0] = record0
+    # }}}
+
+    def build_exth(self): # {{{
+        oeb = self.oeb
+        exth = StringIO()
+        nrecs = 0
+        for term in oeb.metadata:
+            if term not in EXTH_CODES: continue
+            code = EXTH_CODES[term]
+            items = oeb.metadata[term]
+            if term == 'creator':
+                if self.prefer_author_sort:
+                    creators = [normalize(unicode(c.file_as or c)) for c in items]
+                else:
+                    creators = [normalize(unicode(c)) for c in items]
+                items = ['; '.join(creators)]
+            for item in items:
+                data = self.COLLAPSE_RE.sub(' ', normalize(unicode(item)))
+                if term == 'identifier':
+                    if data.lower().startswith('urn:isbn:'):
+                        data = data[9:]
+                    elif item.scheme.lower() == 'isbn':
+                        pass
+                    else:
+                        continue
+                data = data.encode('utf-8')
+                exth.write(pack(b'>II', code, len(data) + 8))
+                exth.write(data)
+                nrecs += 1
+            if term == 'rights' :
+                try:
+                    rights = normalize(unicode(oeb.metadata.rights[0])).encode('utf-8')
+                except:
+                    rights = b'Unknown'
+                exth.write(pack(b'>II', EXTH_CODES['rights'], len(rights) + 8))
+                exth.write(rights)
+                nrecs += 1
+
+        # Write UUID as ASIN
+        uuid = None
+        from calibre.ebooks.oeb.base import OPF
+        for x in oeb.metadata['identifier']:
+            if (x.get(OPF('scheme'), None).lower() == 'uuid' or
+                    unicode(x).startswith('urn:uuid:')):
+                uuid = unicode(x).split(':')[-1]
+                break
+        if uuid is None:
+            from uuid import uuid4
+            uuid = str(uuid4())
+
+        if isinstance(uuid, unicode):
+            uuid = uuid.encode('utf-8')
+        exth.write(pack(b'>II', 113, len(uuid) + 8))
+        exth.write(uuid)
+        nrecs += 1
+
+        # Write cdetype
+        if not self.opts.mobi_periodical:
+            data = b'EBOK'
+            exth.write(pack(b'>II', 501, len(data)+8))
+            exth.write(data)
+            nrecs += 1
+
+        # Add a publication date entry
+        if oeb.metadata['date'] != [] :
+            datestr = str(oeb.metadata['date'][0])
+        elif oeb.metadata['timestamp'] != [] :
+            datestr = str(oeb.metadata['timestamp'][0])
+
+        if datestr is not None:
+            exth.write(pack(b'>II', EXTH_CODES['pubdate'], len(datestr) + 8))
+            exth.write(datestr)
+            nrecs += 1
+        else:
+            raise NotImplementedError("missing date or timestamp needed for mobi_periodical")
+
+        if (oeb.metadata.cover and
+                unicode(oeb.metadata.cover[0]) in oeb.manifest.ids):
+            id = unicode(oeb.metadata.cover[0])
+            item = oeb.manifest.ids[id]
+            href = item.href
+            if href in self.images:
+                index = self.images[href] - 1
+                exth.write(pack(b'>III', 0xc9, 0x0c, index))
+                exth.write(pack(b'>III', 0xcb, 0x0c, 0))
+                nrecs += 2
+                index = self.add_thumbnail(item)
+                if index is not None:
+                    exth.write(pack(b'>III', 0xca, 0x0c, index - 1))
+                    nrecs += 1
+
+        exth = exth.getvalue()
+        trail = len(exth) % 4
+        pad = b'\0' * (4 - trail) # Always pad w/ at least 1 byte
+        exth = [b'EXTH', pack(b'>II', len(exth) + 12, nrecs), exth, pad]
+        return b''.join(exth)
+    # }}}
+
+    def add_thumbnail(self, item):
+        try:
+            data = rescale_image(item.data, dimen=MAX_THUMB_DIMEN,
+                    maxsizeb=MAX_THUMB_SIZE)
+        except IOError:
+            self.oeb.logger.warn('Bad image file %r' % item.href)
+            return None
+        manifest = self.oeb.manifest
+        id, href = manifest.generate('thumbnail', 'thumbnail.jpeg')
+        manifest.add(id, href, 'image/jpeg', data=data)
+        index = len(self.images) + 1
+        self.images[href] = index
+        self.records.append(data)
+        return index
+
+    def write_header(self):
+        title = ascii_filename(unicode(self.oeb.metadata.title[0]))
+        title = title + (b'\0' * (32 - len(title)))
+        now = int(time.time())
+        nrecords = len(self.records)
+        self.write(title, pack(b'>HHIIIIII', 0, 0, now, now, 0, 0, 0, 0),
+            b'BOOK', b'MOBI', pack(b'>IIH', nrecords, 0, nrecords))
+        offset = self.tell() + (8 * nrecords) + 2
+        for i, record in enumerate(self.records):
+            self.write(pack(b'>I', offset), b'\0', pack(b'>I', 2*i)[1:])
+            offset += len(record)
+        self.write(b'\0\0')
+
+    def write_content(self):
+        for record in self.records:
+            self.write(record)
+
+

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.ebooks.oeb.base import (OEB_DOCS, XHTML, XHTML_NS, XML_NS,
+        namespace, prefixname, urlnormalize)
+from calibre.ebooks.mobi.mobiml import MBP_NS
+
+from collections import defaultdict
+from urlparse import urldefrag
+from cStringIO import StringIO
+
+
+class Serializer(object):
+    NSRMAP = {'': None, XML_NS: 'xml', XHTML_NS: '', MBP_NS: 'mbp'}
+
+    def __init__(self, oeb, images, write_page_breaks_after_item=True):
+        '''
+        Write all the HTML markup in oeb into a single in memory buffer
+        containing a single html document with links replaced by offsets into
+        the buffer.
+
+        :param oeb: OEBBook object that encapsulates the document to be
+        processed.
+
+        :param images: Mapping of image hrefs (urlnormalized) to image record
+        indices.
+
+        :param write_page_breaks_after_item: If True a MOBIpocket pagebreak tag
+        is written after every element of the spine in ``oeb``.
+        '''
+        self.oeb = oeb
+        self.images = images
+        self.logger = oeb.logger
+        self.write_page_breaks_after_item = write_page_breaks_after_item
+
+        # Mapping of hrefs (urlnormalized) to the offset in the buffer where
+        # the resource pointed to by the href lives. Used at the end to fill in
+        # the correct values into all filepos="..." links.
+        self.id_offsets = {}
+
+        # Mapping of hrefs (urlnormalized) to a list of offsets into the buffer
+        # where filepos="..." elements are written corresponding to links that
+        # point to the href. This is used at the end to fill in the correct values.
+        self.href_offsets = defaultdict(list)
+
+        # List of offsets in the buffer of non linear items in the spine. These
+        # become uncrossable breaks in the MOBI
+        self.breaks = []
+
+    def __call__(self):
+        '''
+        Return the document serialized as a single UTF-8 encoded bytestring.
+        '''
+        buf = self.buf = StringIO()
+        buf.write(b'<html>')
+        self.serialize_head()
+        self.serialize_body()
+        buf.write(b'</html>')
+        self.fixup_links()
+        return buf.getvalue()
+
+    def serialize_head(self):
+        buf = self.buf
+        buf.write(b'<head>')
+        if len(self.oeb.guide) > 0:
+            self.serialize_guide()
+        buf.write(b'</head>')
+
+    def serialize_guide(self):
+        '''
+        The Kindle decides where to open a book based on the presence of
+        an item in the guide that looks like
+        <reference type="text" title="Start" href="chapter-one.xhtml"/>
+
+        Similarly an item with type="toc" controls where the Goto Table of
+        Contents operation on the kindle goes.
+        '''
+
+        buf = self.buf
+        hrefs = self.oeb.manifest.hrefs
+        buf.write(b'<guide>')
+        for ref in self.oeb.guide.values():
+            path = urldefrag(ref.href)[0]
+            if path not in hrefs or hrefs[path].media_type not in OEB_DOCS:
+                continue
+
+            buf.write(b'<reference type="')
+            if ref.type.startswith('other.') :
+                self.serialize_text(ref.type.replace('other.',''), quot=True)
+            else:
+                self.serialize_text(ref.type, quot=True)
+            buf.write(b'" ')
+            if ref.title is not None:
+                buf.write(b'title="')
+                self.serialize_text(ref.title, quot=True)
+                buf.write(b'" ')
+            self.serialize_href(ref.href)
+            # Space required or won't work, I kid you not
+            buf.write(b' />')
+
+        buf.write(b'</guide>')
+
+    def serialize_href(self, href, base=None):
+        '''
+        Serialize the href attribute of an <a> or <reference> tag. It is
+        serialized as filepos="000000000" and a pointer to its location is
+        stored in self.href_offsets so that the correct value can be filled in
+        at the end.
+        '''
+        hrefs = self.oeb.manifest.hrefs
+        path, frag = urldefrag(urlnormalize(href))
+        if path and base:
+            path = base.abshref(path)
+        if path and path not in hrefs:
+            return False
+        buf = self.buf
+        item = hrefs[path] if path else None
+        if item and item.spine_position is None:
+            return False
+        path = item.href if item else base.href
+        href = '#'.join((path, frag)) if frag else path
+        buf.write(b'filepos=')
+        self.href_offsets[href].append(buf.tell())
+        buf.write(b'0000000000')
+        return True
+
+    def serialize_body(self):
+        '''
+        Serialize all items in the spine of the document. Non linear items are
+        moved to the end.
+        '''
+        buf = self.buf
+        self.anchor_offset = buf.tell()
+        buf.write(b'<body>')
+        self.anchor_offset_kindle = buf.tell()
+        spine = [item for item in self.oeb.spine if item.linear]
+        spine.extend([item for item in self.oeb.spine if not item.linear])
+        for item in spine:
+            self.serialize_item(item)
+        buf.write(b'</body>')
+
+    def serialize_item(self, item):
+        '''
+        Serialize an individual item from the spine of the input document.
+        A reference to this item is stored in self.href_offsets
+        '''
+        buf = self.buf
+        if not item.linear:
+            self.breaks.append(buf.tell() - 1)
+        self.id_offsets[urlnormalize(item.href)] = buf.tell()
+        # Kindle periodical articles are contained in a <div> tag
+        buf.write(b'<div>')
+        for elem in item.data.find(XHTML('body')):
+            self.serialize_elem(elem, item)
+        # Kindle periodical article end marker
+        buf.write(b'<div></div>')
+        if self.write_page_breaks_after_item:
+            buf.write(b'<mbp:pagebreak/>')
+        buf.write(b'</div>')
+        self.anchor_offset = None
+
+    def serialize_elem(self, elem, item, nsrmap=NSRMAP):
+        buf = self.buf
+        if not isinstance(elem.tag, basestring) \
+            or namespace(elem.tag) not in nsrmap:
+                return
+        tag = prefixname(elem.tag, nsrmap)
+        # Previous layers take care of @name
+        id_ = elem.attrib.pop('id', None)
+        if id_:
+            href = '#'.join((item.href, id_))
+            offset = self.anchor_offset or buf.tell()
+            self.id_offsets[urlnormalize(href)] = offset
+        if self.anchor_offset is not None and \
+            tag == 'a' and not elem.attrib and \
+            not len(elem) and not elem.text:
+                return
+        self.anchor_offset = buf.tell()
+        buf.write(b'<')
+        buf.write(tag.encode('utf-8'))
+        if elem.attrib:
+            for attr, val in elem.attrib.items():
+                if namespace(attr) not in nsrmap:
+                    continue
+                attr = prefixname(attr, nsrmap)
+                buf.write(b' ')
+                if attr == 'href':
+                    if self.serialize_href(val, item):
+                        continue
+                elif attr == 'src':
+                    href = urlnormalize(item.abshref(val))
+                    if href in self.images:
+                        index = self.images[href]
+                        buf.write(b'recindex="%05d"' % index)
+                        continue
+                buf.write(attr.encode('utf-8'))
+                buf.write(b'="')
+                self.serialize_text(val, quot=True)
+                buf.write(b'"')
+        buf.write(b'>')
+        if elem.text or len(elem) > 0:
+            if elem.text:
+                self.anchor_offset = None
+                self.serialize_text(elem.text)
+            for child in elem:
+                self.serialize_elem(child, item)
+                if child.tail:
+                    self.anchor_offset = None
+                    self.serialize_text(child.tail)
+        buf.write(b'</%s>' % tag.encode('utf-8'))
+
+    def serialize_text(self, text, quot=False):
+        text = text.replace('&', '&amp;')
+        text = text.replace('<', '&lt;')
+        text = text.replace('>', '&gt;')
+        text = text.replace(u'\u00AD', '') # Soft-hyphen
+        if quot:
+            text = text.replace('"', '&quot;')
+        self.buf.write(text.encode('utf-8'))
+
+    def fixup_links(self):
+        '''
+        Fill in the correct values for all filepos="..." links with the offsets
+        of the linked to content (as stored in id_offsets).
+        '''
+        buf = self.buf
+        id_offsets = self.id_offsets
+        for href, hoffs in self.href_offsets.items():
+            # Iterate over all filepos items
+            if href not in id_offsets:
+                self.logger.warn('Hyperlink target %r not found' % href)
+                # Link to the top of the document, better than just ignoring
+                href, _ = urldefrag(href)
+            if href in self.id_offsets:
+                ioff = self.id_offsets[href]
+                for hoff in hoffs:
+                    buf.seek(hoff)
+                    buf.write(b'%010d' % ioff)
+
+

src/calibre/ebooks/mobi/writer2/utils.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import struct
+
+from calibre.utils.magick.draw import Image, save_cover_data_to, thumbnail
+
+IMAGE_MAX_SIZE = 10 * 1024 * 1024
+
+def decode_hex_number(raw):
+    '''
+    Return a variable length number encoded using hexadecimal encoding. These
+    numbers have the first byte which tells the number of bytes that follow.
+    The bytes that follow are simply the hexadecimal representation of the
+    number.
+
+    :param raw: Raw binary data as a bytestring
+
+    :return: The number and the number of bytes from raw that the number
+    occupies
+    '''
+    length, = struct.unpack(b'>B', raw[0])
+    raw = raw[1:1+length]
+    consumed = length+1
+    return int(raw, 16), consumed
+
+def encode_number_as_hex(num):
+    '''
+    Encode num as a variable length encoded hexadecimal number. Returns the
+    bytestring containing the encoded number. These
+    numbers have the first byte which tells the number of bytes that follow.
+    The bytes that follow are simply the hexadecimal representation of the
+    number.
+    '''
+    num = bytes(hex(num)[2:])
+    ans = bytearray(num)
+    ans.insert(0, len(num))
+    return bytes(ans)
+
+def encint(value, forward=True):
+    '''
+    Some parts of the Mobipocket format encode data as variable-width integers.
+    These integers are represented big-endian with 7 bits per byte in bits 1-7.
+    They may be either forward-encoded, in which case only the first byte has bit 8 set,
+    or backward-encoded, in which case only the last byte has bit 8 set.
+    For example, the number 0x11111 = 0b10001000100010001 would be represented
+    forward-encoded as:
+
+        0x04 0x22 0x91 = 0b100 0b100010 0b10010001
+
+    And backward-encoded as:
+
+        0x84 0x22 0x11 = 0b10000100 0b100010 0b10001
+
+    This function encodes the integer ``value`` as a variable width integer and
+    returns the bytestring corresponding to it.
+
+    If forward is True the bytes returned are suitable for prepending to the
+    output buffer, otherwise they must be append to the output buffer.
+    '''
+    # Encode vwi
+    byts = bytearray()
+    while True:
+        b = value & 0b01111111
+        value >>= 7 # shift value to the right by 7 bits
+        byts.append(b)
+        if value == 0:
+            break
+    byts[0 if forward else -1] |= 0b10000000
+    byts.reverse()
+    return bytes(byts)
+
+def decint(raw, forward=True):
+    '''
+    Read a variable width integer from the bytestring raw and return the
+    integer and the number of bytes read. If forward is True bytes are read
+    from the start of raw, otherwise from the end of raw.
+
+    This function is the inverse of encint above, see its docs for more
+    details.
+    '''
+    val = 0
+    byts = bytearray()
+    for byte in raw if forward else reversed(raw):
+        bnum = ord(byte)
+        byts.append(bnum & 0b01111111)
+        if bnum & 0b10000000:
+            break
+    if not forward:
+        byts.reverse()
+    for byte in byts:
+        val <<= 7 # Shift value to the left by 7 bits
+        val |= byte
+
+    return val, len(byts)
+
+def test_decint(num):
+    for d in (True, False):
+        raw = encint(num, forward=d)
+        sz = len(raw)
+        if (num, sz) != decint(raw, forward=d):
+            raise ValueError('Failed for num %d, forward=%r: %r != %r' % (
+                num, d, (num, sz), decint(raw, forward=d)))
+
+def rescale_image(data, maxsizeb=IMAGE_MAX_SIZE, dimen=None):
+    '''
+    Convert image setting all transparent pixels to white and changing format
+    to JPEG. Ensure the resultant image has a byte size less than
+    maxsizeb.
+
+    If dimen is not None, generate a thumbnail of width=dimen, height=dimen
+
+    Returns the image as a bytestring
+    '''
+    if dimen is not None:
+        data = thumbnail(data, width=dimen, height=dimen,
+                compression_quality=90)[-1]
+    else:
+        # Replace transparent pixels with white pixels and convert to JPEG
+        data = save_cover_data_to(data, 'img.jpg', return_data=True)
+    if len(data) <= maxsizeb:
+        return data
+    orig_data = data
+    img = Image()
+    quality = 95
+
+    img.load(data)
+    while len(data) >= maxsizeb and quality >= 10:
+        quality -= 5
+        img.set_compression_quality(quality)
+        data = img.export('jpg')
+    if len(data) <= maxsizeb:
+        return data
+    orig_data = data
+
+    scale = 0.9
+    while len(data) >= maxsizeb and scale >= 0.05:
+        img = Image()
+        img.load(orig_data)
+        w, h = img.size
+        img.size = (int(scale*w), int(scale*h))
+        img.set_compression_quality(quality)
+        data = img.export('jpg')
+        scale -= 0.05
+    return data
+
+

src/calibre/ebooks/oeb/base.py

@@ -1180,8 +1180,9 @@ class Manifest(object):
                 if memory is None:
                     from calibre.ptempfile import PersistentTemporaryFile
                     pt = PersistentTemporaryFile(suffix='_oeb_base_mem_unloader.img')
+                    with pt:
                         pt.write(self._data)
-                    pt.close()
+                    self.oeb._temp_files.append(pt.name)
                     def loader(*args):
                         with open(pt.name, 'rb') as f:
                             ans = f.read()
@@ -1196,8 +1197,6 @@ class Manifest(object):
                     self._loader = loader2
                 self._data = None
 
-
-
         def __str__(self):
             data = self.data
             if isinstance(data, etree._Element):
@@ -1913,6 +1912,14 @@ class OEBBook(object):
         self.toc = TOC()
         self.pages = PageList()
         self.auto_generated_toc = True
+        self._temp_files = []
+
+    def clean_temp_files(self):
+        for path in self._temp_files:
+            try:
+                os.remove(path)
+            except:
+                pass
 
     @classmethod
     def generate(cls, opts):

src/calibre/ebooks/oeb/iterator.py

@@ -92,7 +92,7 @@ class EbookIterator(object):
         self.config = DynamicConfig(name='iterator')
         ext = os.path.splitext(pathtoebook)[1].replace('.', '').lower()
         ext = re.sub(r'(x{0,1})htm(l{0,1})', 'html', ext)
-        self.ebook_ext = ext
+        self.ebook_ext = ext.replace('original_', '')
 
     def search(self, text, index, backwards=False):
         text = text.lower()

src/calibre/ebooks/oeb/reader.py

@@ -163,6 +163,8 @@ class OEBReader(object):
             if item.media_type in check:
                 try:
                     item.data
+                except KeyboardInterrupt:
+                    raise
                 except:
                     self.logger.exception('Failed to parse content in %s'%
                             item.href)

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -318,7 +318,8 @@ class CSSFlattener(object):
                     for edge in ('top', 'bottom'):
                         cssdict['%s-%s'%(prop, edge)] = '0pt'
             if self.context.insert_blank_line:
-                cssdict['margin-top'] = cssdict['margin-bottom'] = '0.5em'
+                cssdict['margin-top'] = cssdict['margin-bottom'] = \
+                    '%fem'%self.context.insert_blank_line_size
             if self.context.remove_paragraph_spacing:
                 cssdict['text-indent'] =  "%1.1fem" % self.context.remove_paragraph_spacing_indent_size
 

src/calibre/ebooks/oeb/transforms/guide.py

@@ -36,5 +36,8 @@ class Clean(object):
             href = urldefrag(self.oeb.guide[x].href)[0]
             if x.lower() not in ('cover', 'titlepage', 'masthead', 'toc',
                     'title-page', 'copyright-page', 'start'):
+                item = self.oeb.guide[x]
+                if item.title and item.title.lower() == 'start':
+                    continue
                 self.oeb.guide.remove(x)
 

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -45,9 +45,10 @@ body > .calibre_toc_block {
     }
 
 class HTMLTOCAdder(object):
-    def __init__(self, title=None, style='nested'):
+    def __init__(self, title=None, style='nested', position='end'):
         self.title = title
         self.style = style
+        self.position = position
 
     @classmethod
     def config(cls, cfg):
@@ -98,7 +99,10 @@ class HTMLTOCAdder(object):
         self.add_toc_level(body, oeb.toc)
         id, href = oeb.manifest.generate('contents', 'contents.xhtml')
         item = oeb.manifest.add(id, href, XHTML_MIME, data=contents)
+        if self.position == 'end':
             oeb.spine.add(item, linear=False)
+        else:
+            oeb.spine.insert(0, item, linear=True)
         oeb.guide.add('toc', 'Table of Contents', href)
 
     def add_toc_level(self, elem, toc):

src/calibre/ebooks/oeb/transforms/metadata.py

@@ -47,15 +47,19 @@ def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False):
         m.add('series', mi.series)
     elif override_input_metadata:
         m.clear('series')
-    if not mi.is_null('isbn'):
+    identifiers = mi.get_identifiers()
+    set_isbn = False
+    for typ, val in identifiers.iteritems():
         has = False
+        if typ.lower() == 'isbn':
+            set_isbn = True
         for x in m.identifier:
-            if x.scheme.lower() == 'isbn':
-                x.content = mi.isbn
+            if x.scheme.lower() == typ.lower():
+                x.content = val
                 has = True
         if not has:
-            m.add('identifier', mi.isbn, scheme='ISBN')
-    elif override_input_metadata:
+            m.add('identifier', val, scheme=typ.upper())
+    if override_input_metadata and not set_isbn:
         m.filter('identifier', lambda x: x.scheme.lower() == 'isbn')
     if not mi.is_null('language'):
         m.clear('language')

src/calibre/ebooks/snb/snbfile.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 import sys, struct, zlib, bz2, os
 
 from calibre import guess_type
+from calibre.utils.filenames import ascii_filename
 
 class FileStream:
     def IsBinary(self):
@@ -156,6 +157,8 @@ class SNBFile:
         f.fileSize = os.path.getsize(os.path.join(tdir,fileName))
         f.fileBody = open(os.path.join(tdir,fileName), 'rb').read()
         f.fileName = fileName.replace(os.sep, '/')
+        if isinstance(f.fileName, unicode):
+            f.fileName = ascii_filename(f.fileName).encode('ascii')
         self.files.append(f)
 
     def AppendBinary(self, fileName, tdir):
@@ -164,6 +167,8 @@ class SNBFile:
         f.fileSize = os.path.getsize(os.path.join(tdir,fileName))
         f.fileBody = open(os.path.join(tdir,fileName), 'rb').read()
         f.fileName = fileName.replace(os.sep, '/')
+        if isinstance(f.fileName, unicode):
+            f.fileName = ascii_filename(f.fileName).encode('ascii')
         self.files.append(f)
 
     def GetFileStream(self, fileName):

src/calibre/gui2/__init__.py

@@ -15,7 +15,6 @@ APP_UID  = 'libprs500'
 from calibre.constants import (islinux, iswindows, isbsd, isfrozen, isosx,
         config_dir)
 from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig
-from calibre.utils.localization import set_qt_translator
 from calibre.ebooks.metadata import MetaInformation
 from calibre.utils.date import UNDEFINED_DATE
 
@@ -631,6 +630,22 @@ class ResizableDialog(QDialog):
         nw = min(self.width(), nw)
         self.resize(nw, nh)
 
+class Translator(QTranslator):
+    '''
+    Translator to load translations for strings in Qt from the calibre
+    translations. Does not support advanced features of Qt like disambiguation
+    and plural forms.
+    '''
+
+    def translate(self, *args, **kwargs):
+        try:
+            src = unicode(args[1])
+        except:
+            return u''
+        t = _
+        return t(src)
+
+
 gui_thread = None
 
 qt_app = None
@@ -677,8 +692,7 @@ class Application(QApplication):
     def load_translations(self):
         if self._translator is not None:
             self.removeTranslator(self._translator)
-        self._translator = QTranslator(self)
-        if set_qt_translator(self._translator):
+        self._translator = Translator(self)
         self.installTranslator(self._translator)
 
     def event(self, e):

src/calibre/gui2/actions/convert.py

@@ -12,7 +12,7 @@ from PyQt4.Qt import QModelIndex, QMenu
 
 from calibre.gui2 import error_dialog, Dispatcher
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebook
-from calibre.utils.config import prefs
+from calibre.utils.config import prefs, tweaks
 from calibre.gui2.actions import InterfaceAction
 from calibre.customize.ui import plugin_for_input_format
 
@@ -118,6 +118,8 @@ class ConvertAction(InterfaceAction):
     def queue_convert_jobs(self, jobs, changed, bad, rows, previous,
             converted_func, extra_job_args=[]):
         for func, args, desc, fmt, id, temp_files in jobs:
+            func, _, same_fmt = func.partition(':')
+            same_fmt = same_fmt == 'same_fmt'
             input_file = args[0]
             input_fmt = os.path.splitext(input_file)[1]
             core_usage = 1
@@ -131,6 +133,7 @@ class ConvertAction(InterfaceAction):
                 job = self.gui.job_manager.run_job(Dispatcher(converted_func),
                                             func, args=args, description=desc,
                                             core_usage=core_usage)
+                job.conversion_of_same_fmt = same_fmt
                 args = [temp_files, fmt, id]+extra_job_args
                 self.conversion_jobs[job] = tuple(args)
 
@@ -166,14 +169,18 @@ class ConvertAction(InterfaceAction):
             if job.failed:
                 self.gui.job_exception(job)
                 return
+            same_fmt = getattr(job, 'conversion_of_same_fmt', False)
             fmtf = temp_files[-1].name
             if os.stat(fmtf).st_size < 1:
                 raise Exception(_('Empty output file, '
                     'probably the conversion process crashed'))
 
+            db = self.gui.current_db
+            if same_fmt and tweaks['save_original_format']:
+                db.save_original_format(book_id, fmt, notify=False)
+
             with open(temp_files[-1].name, 'rb') as data:
-                self.gui.library_view.model().db.add_format(book_id, \
-                    fmt, data, index_is_id=True)
+                db.add_format(book_id, fmt, data, index_is_id=True)
             self.gui.status_bar.show_message(job.description + \
                     (' completed'), 2000)
         finally:

src/calibre/gui2/actions/delete.py

@@ -81,7 +81,7 @@ class MultiDeleter(QObject):
 class DeleteAction(InterfaceAction):
 
     name = 'Remove Books'
-    action_spec = (_('Remove books'), 'trash.png', None, _('Del'))
+    action_spec = (_('Remove books'), 'trash.png', None, 'Del')
     action_type = 'current'
 
     def genesis(self):

src/calibre/gui2/actions/view.py

@@ -128,7 +128,8 @@ class ViewAction(InterfaceAction):
             self.gui.unsetCursor()
 
     def _view_file(self, name):
-        ext = os.path.splitext(name)[1].upper().replace('.', '')
+        ext = os.path.splitext(name)[1].upper().replace('.',
+                '').replace('ORIGINAL_', '')
         viewer = 'lrfviewer' if ext == 'LRF' else 'ebook-viewer'
         internal = ext in config['internally_viewed_formats']
         self._launch_viewer(name, viewer, internal)

src/calibre/gui2/convert/look_and_feel.py

@@ -24,7 +24,10 @@ class LookAndFeelWidget(Widget, Ui_Form):
                     'font_size_mapping', 'line_height', 'minimum_line_height',
                     'linearize_tables', 'smarten_punctuation',
                     'disable_font_rescaling', 'insert_blank_line',
-                    'remove_paragraph_spacing', 'remove_paragraph_spacing_indent_size','input_encoding',
+                    'remove_paragraph_spacing',
+                    'remove_paragraph_spacing_indent_size',
+                    'insert_blank_line_size',
+                    'input_encoding',
                     'asciiize', 'keep_ligatures']
                 )
         for val, text in [

src/calibre/gui2/convert/look_and_feel.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>600</width>
+    <width>642</width>
     <height>500</height>
    </rect>
   </property>
@@ -31,7 +31,7 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="1" colspan="2">
+   <item row="1" column="1">
     <widget class="QDoubleSpinBox" name="opt_base_font_size">
      <property name="suffix">
       <string> pt</string>
@@ -97,6 +97,29 @@
      </item>
     </layout>
    </item>
+   <item row="3" column="0">
+    <widget class="QLabel" name="label_6">
+     <property name="text">
+      <string>Minimum &amp;line height:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_minimum_line_height</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1">
+    <widget class="QDoubleSpinBox" name="opt_minimum_line_height">
+     <property name="suffix">
+      <string> %</string>
+     </property>
+     <property name="decimals">
+      <number>1</number>
+     </property>
+     <property name="maximum">
+      <double>900.000000000000000</double>
+     </property>
+    </widget>
+   </item>
    <item row="4" column="0">
     <widget class="QLabel" name="label">
      <property name="text">
@@ -107,7 +130,7 @@
      </property>
     </widget>
    </item>
-   <item row="4" column="1" colspan="2">
+   <item row="4" column="1">
     <widget class="QDoubleSpinBox" name="opt_line_height">
      <property name="suffix">
       <string> pt</string>
@@ -127,6 +150,13 @@
      </property>
     </widget>
    </item>
+   <item row="5" column="1" colspan="2">
+    <widget class="EncodingComboBox" name="opt_input_encoding">
+     <property name="editable">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
    <item row="6" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_remove_paragraph_spacing">
      <property name="text">
@@ -134,19 +164,70 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="2" colspan="2">
-    <layout class="QHBoxLayout" name="horizontalLayout_2">
-     <item>
-      <widget class="QLabel" name="label_4">
+   <item row="7" column="0" colspan="2">
+    <widget class="QCheckBox" name="opt_insert_blank_line">
      <property name="text">
-        <string>Indent size:</string>
-       </property>
-       <property name="alignment">
-        <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+      <string>Insert &amp;blank line between paragraphs</string>
      </property>
     </widget>
    </item>
-     <item>
+   <item row="7" column="4">
+    <widget class="QDoubleSpinBox" name="opt_insert_blank_line_size">
+     <property name="suffix">
+      <string> em</string>
+     </property>
+     <property name="decimals">
+      <number>1</number>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>Text &amp;justification:</string>
+     </property>
+     <property name="buddy">
+      <cstring>opt_change_justification</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="2" colspan="3">
+    <widget class="QComboBox" name="opt_change_justification"/>
+   </item>
+   <item row="9" column="0">
+    <widget class="QCheckBox" name="opt_linearize_tables">
+     <property name="text">
+      <string>&amp;Linearize tables</string>
+     </property>
+    </widget>
+   </item>
+   <item row="9" column="1" colspan="4">
+    <widget class="QCheckBox" name="opt_asciiize">
+     <property name="text">
+      <string>&amp;Transliterate unicode characters to ASCII</string>
+     </property>
+    </widget>
+   </item>
+   <item row="10" column="1" colspan="2">
+    <widget class="QCheckBox" name="opt_keep_ligatures">
+     <property name="text">
+      <string>Keep &amp;ligatures</string>
+     </property>
+    </widget>
+   </item>
+   <item row="12" column="0" colspan="5">
+    <widget class="QGroupBox" name="groupBox">
+     <property name="title">
+      <string>Extra &amp;CSS</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_3">
+      <item row="0" column="0">
+       <widget class="QTextEdit" name="opt_extra_css"/>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="6" column="4">
     <widget class="QDoubleSpinBox" name="opt_remove_paragraph_spacing_indent_size">
      <property name="toolTip">
       <string>&lt;p&gt;When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent.</string>
@@ -159,58 +240,6 @@
      </property>
     </widget>
    </item>
-    </layout>
-   </item>
-   <item row="7" column="0">
-    <widget class="QLabel" name="label_5">
-     <property name="text">
-      <string>Text justification:</string>
-     </property>
-    </widget>
-   </item>
-   <item row="8" column="0">
-    <widget class="QCheckBox" name="opt_linearize_tables">
-     <property name="text">
-      <string>&amp;Linearize tables</string>
-     </property>
-    </widget>
-   </item>
-   <item row="11" column="0" colspan="4">
-    <widget class="QGroupBox" name="groupBox">
-     <property name="title">
-      <string>Extra &amp;CSS</string>
-     </property>
-     <layout class="QGridLayout" name="gridLayout_3">
-      <item row="0" column="0">
-       <widget class="QTextEdit" name="opt_extra_css"/>
-      </item>
-     </layout>
-    </widget>
-   </item>
-   <item row="7" column="2" colspan="2">
-    <widget class="QComboBox" name="opt_change_justification"/>
-   </item>
-   <item row="8" column="1" colspan="3">
-    <widget class="QCheckBox" name="opt_asciiize">
-     <property name="text">
-      <string>&amp;Transliterate unicode characters to ASCII</string>
-     </property>
-    </widget>
-   </item>
-   <item row="9" column="0">
-    <widget class="QCheckBox" name="opt_insert_blank_line">
-     <property name="text">
-      <string>Insert &amp;blank line</string>
-     </property>
-    </widget>
-   </item>
-   <item row="9" column="1" colspan="2">
-    <widget class="QCheckBox" name="opt_keep_ligatures">
-     <property name="text">
-      <string>Keep &amp;ligatures</string>
-     </property>
-    </widget>
-   </item>
    <item row="10" column="0">
     <widget class="QCheckBox" name="opt_smarten_punctuation">
      <property name="text">
@@ -218,33 +247,29 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="0">
-    <widget class="QLabel" name="label_6">
+   <item row="6" column="3">
+    <widget class="QLabel" name="label_4">
      <property name="text">
-      <string>Minimum &amp;line height:</string>
+      <string>&amp;Indent size:</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
      </property>
      <property name="buddy">
-      <cstring>opt_minimum_line_height</cstring>
+      <cstring>opt_remove_paragraph_spacing_indent_size</cstring>
      </property>
     </widget>
    </item>
-   <item row="3" column="1" colspan="2">
-    <widget class="QDoubleSpinBox" name="opt_minimum_line_height">
-     <property name="suffix">
-      <string> %</string>
+   <item row="7" column="3">
+    <widget class="QLabel" name="label_7">
+     <property name="text">
+      <string>&amp;Line size:</string>
      </property>
-     <property name="decimals">
-      <number>1</number>
+     <property name="alignment">
+      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
      </property>
-     <property name="maximum">
-      <double>900.000000000000000</double>
-     </property>
-    </widget>
-   </item>
-   <item row="5" column="1" colspan="3">
-    <widget class="EncodingComboBox" name="opt_input_encoding">
-     <property name="editable">
-      <bool>true</bool>
+     <property name="buddy">
+      <cstring>opt_insert_blank_line_size</cstring>
      </property>
     </widget>
    </item>

src/calibre/gui2/convert/mobi_output.py

@@ -24,7 +24,7 @@ class PluginWidget(Widget, Ui_Form):
     def __init__(self, parent, get_option, get_help, db=None, book_id=None):
         Widget.__init__(self, parent,
                 ['prefer_author_sort', 'rescale_images', 'toc_title',
-                    'mobi_ignore_margins',
+                    'mobi_ignore_margins', 'mobi_toc_at_start',
                 'dont_compress', 'no_inline_toc', 'masthead_font','personal_doc']
                 )
         from calibre.utils.fonts import fontconfig

src/calibre/gui2/convert/mobi_output.ui

@@ -27,21 +27,21 @@
    <item row="1" column="1">
     <widget class="QLineEdit" name="opt_toc_title"/>
    </item>
-   <item row="2" column="0" colspan="2">
+   <item row="4" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_rescale_images">
      <property name="text">
       <string>Rescale images for &amp;Palm devices</string>
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="2">
+   <item row="5" column="0" colspan="2">
     <widget class="QCheckBox" name="opt_prefer_author_sort">
      <property name="text">
       <string>Use author &amp;sort for author</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0">
+   <item row="6" column="0">
     <widget class="QCheckBox" name="opt_dont_compress">
      <property name="text">
       <string>Disable compression of the file contents</string>
@@ -55,7 +55,7 @@
      </property>
     </widget>
    </item>
-   <item row="6" column="0" colspan="2">
+   <item row="8" column="0" colspan="2">
     <widget class="QGroupBox" name="groupBox">
      <property name="title">
       <string>Kindle options</string>
@@ -101,7 +101,7 @@
      </layout>
     </widget>
    </item>
-   <item row="7" column="0">
+   <item row="9" column="0">
     <spacer name="verticalSpacer_2">
      <property name="orientation">
       <enum>Qt::Vertical</enum>
@@ -114,7 +114,14 @@
      </property>
     </spacer>
    </item>
-   <item row="5" column="0">
+   <item row="2" column="0" colspan="2">
+    <widget class="QCheckBox" name="opt_mobi_toc_at_start">
+     <property name="text">
+      <string>Put generated Table of Contents at &amp;start of book instead of end</string>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="0">
     <widget class="QCheckBox" name="opt_mobi_ignore_margins">
      <property name="text">
       <string>Ignore &amp;margins</string>

src/calibre/gui2/convert/regex_builder.py

@@ -7,8 +7,8 @@ __docformat__ = 'restructuredtext en'
 import re, os
 
 from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal
-from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \
-                        QBrush, QTextCursor, QTextEdit
+from PyQt4.QtGui import (QDialog, QWidget, QDialogButtonBox,
+        QBrush, QTextCursor, QTextEdit)
 
 from calibre.gui2.convert.regex_builder_ui import Ui_RegexBuilder
 from calibre.gui2.convert.xexp_edit_ui import Ui_Form as Ui_Edit
@@ -16,6 +16,7 @@ from calibre.gui2 import error_dialog, choose_files
 from calibre.ebooks.oeb.iterator import EbookIterator
 from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
+from calibre.constants import iswindows
 
 class RegexBuilder(QDialog, Ui_RegexBuilder):
 
@@ -134,8 +135,18 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
                          _('Cannot build regex using the GUI builder without a book.'),
                          show=True)
             return False
+        try:
             fpath = db.format(book_id, format, index_is_id=True,
                 as_path=True)
+        except OSError:
+            if iswindows:
+                import traceback
+                error_dialog(self, _('Could not open file'),
+                    _('Could not open the file, do you have it open in'
+                        ' another program?'), show=True,
+                    det_msg=traceback.format_exc())
+                return False
+            raise
         try:
             self.open_book(fpath)
         finally:

src/calibre/gui2/custom_column_widgets.py

@@ -723,6 +723,7 @@ class BulkSeries(BulkBase):
         layout.addWidget(self.force_number)
         self.series_start_number = QSpinBox(parent)
         self.series_start_number.setMinimum(1)
+        self.series_start_number.setMaximum(9999999)
         self.series_start_number.setProperty("value", 1)
         layout.addWidget(self.series_start_number)
         layout.addItem(QSpacerItem(20, 10, QSizePolicy.Expanding, QSizePolicy.Minimum))

src/calibre/gui2/dialogs/jobs.ui

@@ -29,9 +29,6 @@
      <property name="alternatingRowColors">
       <bool>true</bool>
      </property>
-     <property name="selectionMode">
-      <enum>QAbstractItemView::SingleSelection</enum>
-     </property>
      <property name="selectionBehavior">
       <enum>QAbstractItemView::SelectRows</enum>
      </property>
@@ -46,7 +43,7 @@
    <item>
     <widget class="QPushButton" name="kill_button">
      <property name="text">
-      <string>&amp;Stop selected job</string>
+      <string>&amp;Stop selected jobs</string>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -749,15 +749,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         val = self.s_r_do_regexp(mi)
         val = self.s_r_do_destination(mi, val)
         if dfm['is_multiple']:
-            if dest == 'authors' and len(val) == 0:
-                error_dialog(self, _('Search/replace invalid'),
-                             _('Authors cannot be set to the empty string. '
-                               'Book title %s not processed')%mi.title,
-                             show=True)
-                return
-            # convert the colon-separated pair strings back into a dict, which
-            # is what set_identifiers wants
             if dfm['is_csp']:
+                # convert the colon-separated pair strings back into a dict,
+                # which is what set_identifiers wants
                 dst_id_type = unicode(self.s_r_dst_ident.text())
                 if dst_id_type:
                     v = ''.join(val)
@@ -769,11 +763,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         else:
             val = self.s_r_replace_mode_separator().join(val)
             if dest == 'title' and len(val) == 0:
-                error_dialog(self, _('Search/replace invalid'),
-                             _('Title cannot be set to the empty string. '
-                               'Book title %s not processed')%mi.title,
-                             show=True)
-                return
+                val = _('Unknown')
 
         if dfm['is_custom']:
             extra = self.db.get_custom_extra(id, label=dfm['label'], index_is_id=True)

src/calibre/gui2/dialogs/tag_categories.py

@@ -260,6 +260,7 @@ class TagCategories(QDialog, Ui_TagCategories):
             self.applied_items = [cat[2] for cat in self.categories.get(self.current_cat_name, [])]
         else:
             self.applied_items = []
+        self.applied_items.sort(key=lambda x:sort_key(self.all_items[x].name))
         self.display_filtered_categories(None)
 
     def accept(self):

src/calibre/gui2/jobs.py

@@ -172,8 +172,9 @@ class JobManager(QAbstractTableModel): # {{{
                     if job.is_finished:
                         self.job_done.emit(len(self.unfinished_jobs()))
             if needs_reset:
+                self.layoutAboutToBeChanged.emit()
                 self.jobs.sort()
-                self.reset()
+                self.layoutChanged.emit()
             else:
                 for job in jobs:
                     idx = self.jobs.index(job)
@@ -267,7 +268,8 @@ class JobManager(QAbstractTableModel): # {{{
 # }}}
 
 # Jobs UI {{{
-class ProgressBarDelegate(QAbstractItemDelegate):
+
+class ProgressBarDelegate(QAbstractItemDelegate): # {{{
 
     def sizeHint(self, option, index):
         return QSize(120, 30)
@@ -284,8 +286,9 @@ class ProgressBarDelegate(QAbstractItemDelegate):
         opts.progress = percent
         opts.text = QString(_('Unavailable') if percent == 0 else '%d%%'%percent)
         QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter)
+# }}}
 
-class DetailView(QDialog, Ui_Dialog):
+class DetailView(QDialog, Ui_Dialog): # {{{
 
     def __init__(self, parent, job):
         QDialog.__init__(self, parent)
@@ -318,8 +321,9 @@ class DetailView(QDialog, Ui_Dialog):
             self.next_pos = f.tell()
             if more:
                 self.log.appendPlainText(more.decode('utf-8', 'replace'))
+# }}}
 
-class JobsButton(QFrame):
+class JobsButton(QFrame): # {{{
 
     def __init__(self, horizontal=False, size=48, parent=None):
         QFrame.__init__(self, parent)
@@ -404,6 +408,7 @@ class JobsButton(QFrame):
             self.stop()
             QCoreApplication.instance().alert(self, 5000)
 
+# }}}
 
 class JobsDialog(QDialog, Ui_JobsDialog):
 
@@ -446,7 +451,6 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         except:
             pass
 
-
     def show_job_details(self, index):
         row = index.row()
         job = self.jobs_view.model().row_to_job(row)
@@ -455,18 +459,23 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         d.timer.stop()
 
     def show_details(self, *args):
-        for index in self.jobs_view.selectedIndexes():
+        index = self.jobs_view.currentIndex()
+        if index.isValid():
             self.show_job_details(index)
-            return
 
     def kill_job(self, *args):
-        if question_dialog(self, _('Are you sure?'), _('Do you really want to stop the selected job?')):
-            for index in self.jobs_view.selectionModel().selectedRows():
-                row = index.row()
+        rows = [index.row() for index in
+                self.jobs_view.selectionModel().selectedRows()]
+        if question_dialog(self, _('Are you sure?'),
+                ngettext('Do you really want to stop the selected job?',
+                    'Do you really want to stop all the selected jobs?',
+                    len(rows))):
+            for row in rows:
                 self.model.kill_job(row, self)
 
     def kill_all_jobs(self, *args):
-        if question_dialog(self, _('Are you sure?'), _('Do you really want to stop all non-device jobs?')):
+        if question_dialog(self, _('Are you sure?'),
+                _('Do you really want to stop all non-device jobs?')):
             self.model.kill_all_jobs()
 
     def closeEvent(self, e):

src/calibre/gui2/library/delegates.py

@@ -16,7 +16,7 @@ from PyQt4.Qt import (QColor, Qt, QModelIndex, QSize, QApplication,
 
 from calibre.gui2 import UNDEFINED_QDATE, error_dialog
 from calibre.gui2.widgets import EnLineEdit
-from calibre.gui2.complete import MultiCompleteLineEdit
+from calibre.gui2.complete import MultiCompleteLineEdit, MultiCompleteComboBox
 from calibre.utils.date import now, format_date
 from calibre.utils.config import tweaks
 from calibre.utils.formatter import validation_formatter
@@ -166,13 +166,26 @@ class TextDelegate(QStyledItemDelegate): # {{{
 
     def createEditor(self, parent, option, index):
         if self.auto_complete_function:
-            editor = MultiCompleteLineEdit(parent)
+            editor = MultiCompleteComboBox(parent)
             editor.set_separator(None)
             complete_items = [i[1] for i in self.auto_complete_function()]
             editor.update_items_cache(complete_items)
+            for item in sorted(complete_items, key=sort_key):
+                editor.addItem(item)
+            ct = index.data(Qt.DisplayRole).toString()
+            editor.setEditText(ct)
+            editor.lineEdit().selectAll()
         else:
             editor = EnLineEdit(parent)
         return editor
+
+    def setModelData(self, editor, model, index):
+        if isinstance(editor, MultiCompleteComboBox):
+            val = editor.lineEdit().text()
+            model.setData(index, QVariant(val), Qt.EditRole)
+        else:
+            QStyledItemDelegate.setModelData(self, editor, model, index)
+
 #}}}
 
 class CompleteDelegate(QStyledItemDelegate): # {{{
@@ -188,7 +201,7 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
     def createEditor(self, parent, option, index):
         if self.db and hasattr(self.db, self.items_func_name):
             col = index.model().column_map[index.column()]
-            editor = MultiCompleteLineEdit(parent)
+            editor = MultiCompleteComboBox(parent)
             editor.set_separator(self.sep)
             editor.set_space_before_sep(self.space_before_sep)
             if self.sep == '&':
@@ -199,9 +212,21 @@ class CompleteDelegate(QStyledItemDelegate): # {{{
                 all_items = list(self.db.all_custom(
                     label=self.db.field_metadata.key_to_label(col)))
             editor.update_items_cache(all_items)
+            for item in sorted(all_items, key=sort_key):
+                editor.addItem(item)
+            ct = index.data(Qt.DisplayRole).toString()
+            editor.setEditText(ct)
+            editor.lineEdit().selectAll()
         else:
             editor = EnLineEdit(parent)
         return editor
+
+    def setModelData(self, editor, model, index):
+        if isinstance(editor, MultiCompleteComboBox):
+            val = editor.lineEdit().text()
+            model.setData(index, QVariant(val), Qt.EditRole)
+        else:
+            QStyledItemDelegate.setModelData(self, editor, model, index)
 # }}}
 
 class CcDateDelegate(QStyledItemDelegate): # {{{

src/calibre/gui2/metadata/basic_widgets.py

@@ -11,10 +11,10 @@ import textwrap, re, os
 
 from PyQt4.Qt import (Qt, QDateEdit, QDate, pyqtSignal, QMessageBox,
     QIcon, QToolButton, QWidget, QLabel, QGridLayout, QApplication,
-    QDoubleSpinBox, QListWidgetItem, QSize, QPixmap, QDialog,
-    QPushButton, QSpinBox, QLineEdit, QSizePolicy, QDialogButtonBox)
+    QDoubleSpinBox, QListWidgetItem, QSize, QPixmap, QDialog, QMenu,
+    QPushButton, QSpinBox, QLineEdit, QSizePolicy, QDialogButtonBox, QAction)
 
-from calibre.gui2.widgets import EnLineEdit, FormatList, ImageView
+from calibre.gui2.widgets import EnLineEdit, FormatList as _FormatList, ImageView
 from calibre.gui2.complete import MultiCompleteLineEdit, MultiCompleteComboBox
 from calibre.utils.icu import sort_key
 from calibre.utils.config import tweaks, prefs
@@ -33,6 +33,7 @@ from calibre.gui2.comments_editor import Editor
 from calibre.library.comments import comments_to_html
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.utils.icu import strcmp
+from calibre.ptempfile import PersistentTemporaryFile
 
 def save_dialog(parent, title, msg, det_msg=''):
     d = QMessageBox(parent)
@@ -572,7 +573,9 @@ class BuddyLabel(QLabel): # {{{
         self.setAlignment(Qt.AlignRight|Qt.AlignVCenter)
 # }}}
 
-class Format(QListWidgetItem): # {{{
+# Formats {{{
+
+class Format(QListWidgetItem):
 
     def __init__(self, parent, ext, size, path=None, timestamp=None):
         self.path = path
@@ -588,13 +591,52 @@ class Format(QListWidgetItem): # {{{
             self.setToolTip(text)
             self.setStatusTip(text)
 
-# }}}
+class OrigAction(QAction):
 
-class FormatsManager(QWidget): # {{{
+    restore_fmt = pyqtSignal(object)
+
+    def __init__(self, fmt, parent):
+        self.fmt = fmt.replace('ORIGINAL_', '')
+        QAction.__init__(self, _('Restore %s from the original')%self.fmt, parent)
+        self.triggered.connect(self._triggered)
+
+    def _triggered(self):
+        self.restore_fmt.emit(self.fmt)
+
+class FormatList(_FormatList):
+
+    restore_fmt = pyqtSignal(object)
 
     def __init__(self, parent):
+        _FormatList.__init__(self, parent)
+        self.setContextMenuPolicy(Qt.DefaultContextMenu)
+
+    def contextMenuEvent(self, event):
+        originals = [self.item(x).ext.upper() for x in range(self.count())]
+        originals = [x for x in originals if x.startswith('ORIGINAL_')]
+        if not originals:
+            return
+        self.cm = cm = QMenu(self)
+        for fmt in originals:
+            action = OrigAction(fmt, cm)
+            action.restore_fmt.connect(self.restore_fmt)
+            cm.addAction(action)
+        cm.popup(event.globalPos())
+        event.accept()
+
+    def remove_format(self, fmt):
+        for i in range(self.count()):
+            f = self.item(i)
+            if f.ext.upper() == fmt.upper():
+                self.takeItem(i)
+                break
+
+class FormatsManager(QWidget):
+
+    def __init__(self, parent, copy_fmt):
         QWidget.__init__(self, parent)
         self.dialog = parent
+        self.copy_fmt = copy_fmt
         self.changed = False
 
         self.l = l = QGridLayout()
@@ -628,6 +670,7 @@ class FormatsManager(QWidget): # {{{
         self.formats = FormatList(self)
         self.formats.setAcceptDrops(True)
         self.formats.formats_dropped.connect(self.formats_dropped)
+        self.formats.restore_fmt.connect(self.restore_fmt)
         self.formats.delete_format.connect(self.remove_format)
         self.formats.itemDoubleClicked.connect(self.show_format)
         self.formats.setDragDropMode(self.formats.DropOnly)
@@ -640,7 +683,7 @@ class FormatsManager(QWidget): # {{{
         l.addWidget(self.remove_format_button,        2, 2, 1, 1)
         l.addWidget(self.formats,                     0, 1, 3, 1)
 
-
+        self.temp_files = []
 
     def initialize(self, db, id_):
         self.changed = False
@@ -694,6 +737,16 @@ class FormatsManager(QWidget): # {{{
                              [(_('Books'), BOOK_EXTENSIONS)])
         self._add_formats(files)
 
+    def restore_fmt(self, fmt):
+        pt = PersistentTemporaryFile(suffix='_restore_fmt.'+fmt.lower())
+        ofmt = 'ORIGINAL_'+fmt
+        with pt:
+            self.copy_fmt(ofmt, pt)
+        self._add_formats((pt.name,))
+        self.temp_files.append(pt.name)
+        self.changed = True
+        self.formats.remove_format(ofmt)
+
     def _add_formats(self, paths):
         added = False
         if not paths:
@@ -774,6 +827,13 @@ class FormatsManager(QWidget): # {{{
 
     def break_cycles(self):
         self.dialog = None
+        self.copy_fmt = None
+        for name in self.temp_files:
+            try:
+                os.remove(name)
+            except:
+                pass
+        self.temp_files = []
 # }}}
 
 class Cover(ImageView): # {{{

src/calibre/gui2/metadata/single.py

@@ -145,7 +145,7 @@ class MetadataSingleDialogBase(ResizableDialog):
         self.series_index = SeriesIndexEdit(self, self.series)
         self.basic_metadata_widgets.extend([self.series, self.series_index])
 
-        self.formats_manager = FormatsManager(self)
+        self.formats_manager = FormatsManager(self, self.copy_fmt)
         self.basic_metadata_widgets.append(self.formats_manager)
         self.formats_manager.metadata_from_format_button.clicked.connect(
                 self.metadata_from_format)
@@ -240,6 +240,8 @@ class MetadataSingleDialogBase(ResizableDialog):
         else:
             self.view_format.emit(self.book_id, fmt)
 
+    def copy_fmt(self, fmt, f):
+        self.db.copy_format_to(self.book_id, fmt, f, index_is_id=True)
 
     def do_layout(self):
         raise NotImplementedError()

src/calibre/gui2/preferences/look_feel.py

@@ -105,13 +105,18 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         r('cover_flow_queue_length', config, restart_required=True)
 
+        def get_esc_lang(l):
+            if l == 'en':
+                return 'English'
+            return get_language(l)
+
         lang = get_lang()
         if lang is None or lang not in available_translations():
             lang = 'en'
-        items = [(l, get_language(l)) for l in available_translations() \
+        items = [(l, get_esc_lang(l)) for l in available_translations() \
                  if l != lang]
         if lang != 'en':
-            items.append(('en', get_language('en')))
+            items.append(('en', get_esc_lang('en')))
         items.sort(cmp=lambda x, y: cmp(x[1].lower(), y[1].lower()))
         choices = [(y, x) for x, y in items]
         # Default language is the autodetected one

src/calibre/gui2/preferences/toolbar.ui

@@ -14,30 +14,18 @@
    <string>Form</string>
   </property>
   <layout class="QGridLayout" name="gridLayout">
-   <item row="0" column="0">
-    <widget class="QLabel" name="label">
-     <property name="font">
-      <font>
-       <weight>75</weight>
-       <bold>true</bold>
-      </font>
-     </property>
-     <property name="text">
-      <string>Choose the &amp;toolbar to customize:</string>
-     </property>
-     <property name="buddy">
-      <cstring>what</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="0" column="1" colspan="3">
+   <item row="0" column="0" colspan="5">
     <widget class="QComboBox" name="what">
      <property name="font">
       <font>
+       <pointsize>20</pointsize>
        <weight>75</weight>
        <bold>true</bold>
       </font>
      </property>
+     <property name="toolTip">
+      <string>Choose the toolbar to customize</string>
+     </property>
      <property name="sizeAdjustPolicy">
       <enum>QComboBox::AdjustToMinimumContentsLengthWithIcon</enum>
      </property>
@@ -46,7 +34,7 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="0" rowspan="2">
+   <item row="1" column="0" colspan="2">
     <widget class="QGroupBox" name="groupBox">
      <property name="title">
       <string>A&amp;vailable actions</string>
@@ -74,7 +62,67 @@
      </layout>
     </widget>
    </item>
-   <item row="1" column="2" rowspan="2">
+   <item row="1" column="2">
+    <layout class="QVBoxLayout" name="verticalLayout_3">
+     <item>
+      <widget class="QToolButton" name="add_action_button">
+       <property name="toolTip">
+        <string>Add selected actions to toolbar</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset resource="../../../../resources/images.qrc">
+         <normaloff>:/images/forward.png</normaloff>:/images/forward.png</iconset>
+       </property>
+       <property name="iconSize">
+        <size>
+         <width>24</width>
+         <height>24</height>
+        </size>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <spacer name="verticalSpacer">
+       <property name="orientation">
+        <enum>Qt::Vertical</enum>
+       </property>
+       <property name="sizeType">
+        <enum>QSizePolicy::Fixed</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>20</width>
+         <height>40</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QToolButton" name="remove_action_button">
+       <property name="toolTip">
+        <string>Remove selected actions from toolbar</string>
+       </property>
+       <property name="text">
+        <string>...</string>
+       </property>
+       <property name="icon">
+        <iconset resource="../../../../resources/images.qrc">
+         <normaloff>:/images/back.png</normaloff>:/images/back.png</iconset>
+       </property>
+       <property name="iconSize">
+        <size>
+         <width>24</width>
+         <height>24</height>
+        </size>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="1" column="3" colspan="2">
     <widget class="QGroupBox" name="groupBox_2">
      <property name="title">
       <string>&amp;Current actions</string>
@@ -162,66 +210,6 @@
      </layout>
     </widget>
    </item>
-   <item row="1" column="1" rowspan="2">
-    <layout class="QVBoxLayout" name="verticalLayout_3">
-     <item>
-      <widget class="QToolButton" name="add_action_button">
-       <property name="toolTip">
-        <string>Add selected actions to toolbar</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset resource="../../../../resources/images.qrc">
-         <normaloff>:/images/forward.png</normaloff>:/images/forward.png</iconset>
-       </property>
-       <property name="iconSize">
-        <size>
-         <width>24</width>
-         <height>24</height>
-        </size>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <spacer name="verticalSpacer">
-       <property name="orientation">
-        <enum>Qt::Vertical</enum>
-       </property>
-       <property name="sizeType">
-        <enum>QSizePolicy::Fixed</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>20</width>
-         <height>40</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
-      <widget class="QToolButton" name="remove_action_button">
-       <property name="toolTip">
-        <string>Remove selected actions from toolbar</string>
-       </property>
-       <property name="text">
-        <string>...</string>
-       </property>
-       <property name="icon">
-        <iconset resource="../../../../resources/images.qrc">
-         <normaloff>:/images/back.png</normaloff>:/images/back.png</iconset>
-       </property>
-       <property name="iconSize">
-        <size>
-         <width>24</width>
-         <height>24</height>
-        </size>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/store/stores/legimi_plugin.py

@@ -60,8 +60,7 @@ class LegimiStore(BasicStoreConfig, StorePlugin):
                 author = ''.join(data.xpath('.//div[@class="item_entries"]/span[1]/a/text()'))
                 author = re.sub(',','',author)
                 author = re.sub(';',',',author)
-                price = ''.join(data.xpath('.//div[@class="item_entries"]/span[3]/text()'))
-                price = re.sub(r'[^0-9,]*','',price) + ' zł'
+                price = ''.join(data.xpath('.//span[@class="ebook_price"]/text()'))
 
                 counter -= 1
                 

src/calibre/gui2/tag_browser/model.py

@@ -685,42 +685,35 @@ class TagsModel(QAbstractItemModel): # {{{
 
     def handle_user_category_drop(self, on_node, ids, column):
         categories = self.db.prefs.get('user_categories', {})
-        category = categories.get(on_node.category_key[1:], None)
-        if category is None:
+        cat_contents = categories.get(on_node.category_key[1:], None)
+        if cat_contents is None:
             return
+        cat_contents = set([(v, c) for v,c,ign in cat_contents])
+
         fm_src = self.db.metadata_for_field(column)
-        for id in ids:
         label = fm_src['label']
+
+        for id in ids:
             if not fm_src['is_custom']:
                 if label == 'authors':
-                    items = self.db.get_authors_with_ids()
-                    items = [(i[0], i[1].replace('|', ',')) for i in items]
                     value = self.db.authors(id, index_is_id=True)
                     value = [v.replace('|', ',') for v in value.split(',')]
                 elif label == 'publisher':
-                    items = self.db.get_publishers_with_ids()
                     value = self.db.publisher(id, index_is_id=True)
                 elif label == 'series':
-                    items = self.db.get_series_with_ids()
                     value = self.db.series(id, index_is_id=True)
             else:
-                items = self.db.get_custom_items_with_ids(label=label)
                 if fm_src['datatype'] != 'composite':
                     value = self.db.get_custom(id, label=label, index_is_id=True)
                 else:
                     value = self.db.get_property(id, loc=fm_src['rec_index'],
                                                  index_is_id=True)
-            if value is None:
-                return
+            if value:
                 if not isinstance(value, list):
                     value = [value]
-            for val in value:
-                for (v, c, id) in category:
-                    if v == val and c == column:
-                        break
-                else:
-                    category.append([val, column, 0])
-            categories[on_node.category_key[1:]] = category
+                cat_contents |= set([(v, column) for v in value])
+
+        categories[on_node.category_key[1:]] = [[v, c, 0] for v,c in cat_contents]
         self.db.prefs.set('user_categories', categories)
         self.refresh_required.emit()
 

src/calibre/gui2/tools.py

@@ -53,7 +53,9 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                 mi = db.get_metadata(book_id, True)
                 in_file = PersistentTemporaryFile('.'+d.input_format)
                 with in_file:
-                    db.copy_format_to(book_id, d.input_format, in_file,
+                    input_fmt = db.original_fmt(book_id, d.input_format).lower()
+                    same_fmt = input_fmt == d.output_format.lower()
+                    db.copy_format_to(book_id, input_fmt, in_file,
                             index_is_id=True)
 
                 out_file = PersistentTemporaryFile('.' + d.output_format)
@@ -79,7 +81,10 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                     temp_files.append(d.cover_file)
                 args = [in_file.name, out_file.name, recs]
                 temp_files.append(out_file)
-                jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files))
+                func = 'gui_convert_override'
+                if same_fmt:
+                    func += ':same_fmt'
+                jobs.append((func, args, desc, d.output_format.upper(), book_id, temp_files))
 
                 changed = True
                 d.break_cycles()
@@ -144,10 +149,12 @@ class QueueBulk(QProgressDialog):
 
         try:
             input_format = get_input_format_for_book(self.db, book_id, None)[0]
+            input_fmt = self.db.original_fmt(book_id, input_format).lower()
+            same_fmt = input_fmt == self.output_format.lower()
             mi, opf_file = create_opf_file(self.db, book_id)
             in_file = PersistentTemporaryFile('.'+input_format)
             with in_file:
-                self.db.copy_format_to(book_id, input_format, in_file,
+                self.db.copy_format_to(book_id, input_fmt, in_file,
                         index_is_id=True)
 
             out_file = PersistentTemporaryFile('.' + self.output_format)
@@ -192,7 +199,10 @@ class QueueBulk(QProgressDialog):
 
             args = [in_file.name, out_file.name, lrecs]
             temp_files.append(out_file)
-            self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files))
+            func = 'gui_convert_override'
+            if same_fmt:
+                func += ':same_fmt'
+            self.jobs.append((func, args, desc, self.output_format.upper(), book_id, temp_files))
 
             self.changed = True
             self.setValue(self.i)

src/calibre/gui2/viewer/main.py

@@ -661,6 +661,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
     def save_current_position(self):
         if not self.get_remember_current_page_opt():
             return
+        if hasattr(self, 'current_index'):
             try:
                 pos = self.view.bookmark()
                 bookmark = '%d#%s'%(self.current_index, pos)

src/calibre/gui2/wizard/__init__.py

@@ -604,16 +604,21 @@ class LibraryPage(QWizardPage, LibraryUI):
     def init_languages(self):
         self.language.blockSignals(True)
         self.language.clear()
-        from calibre.utils.localization import available_translations, \
-            get_language, get_lang
+        from calibre.utils.localization import (available_translations,
+            get_language, get_lang)
         lang = get_lang()
         if lang is None or lang not in available_translations():
             lang = 'en'
-        self.language.addItem(get_language(lang), QVariant(lang))
-        items = [(l, get_language(l)) for l in available_translations() \
+        def get_esc_lang(l):
+            if l == 'en':
+                return 'English'
+            return get_language(l)
+
+        self.language.addItem(get_esc_lang(lang), QVariant(lang))
+        items = [(l, get_esc_lang(l)) for l in available_translations()
                  if l != lang]
         if lang != 'en':
-            items.append(('en', get_language('en')))
+            items.append(('en', get_esc_lang('en')))
         items.sort(cmp=lambda x, y: cmp(x[1], y[1]))
         for item in items:
             self.language.addItem(item[1], QVariant(item[0]))

src/calibre/library/database2.py

@@ -1312,6 +1312,23 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.notify('metadata', [id])
         return True
 
+    def save_original_format(self, book_id, fmt, notify=True):
+        fmt = fmt.upper()
+        if 'ORIGINAL' in fmt:
+            raise ValueError('Cannot save original of an original fmt')
+        opath = self.format_abspath(book_id, fmt, index_is_id=True)
+        if opath is None:
+            return False
+        nfmt = 'ORIGINAL_'+fmt
+        with lopen(opath, 'rb') as f:
+            return self.add_format(book_id, nfmt, f, index_is_id=True, notify=notify)
+
+    def original_fmt(self, book_id, fmt):
+        fmt = fmt
+        nfmt = ('ORIGINAL_%s'%fmt).upper()
+        opath = self.format_abspath(book_id, nfmt, index_is_id=True)
+        return fmt if opath is None else nfmt
+
     def delete_book(self, id, notify=True, commit=True, permanent=False):
         '''
         Removes book from the result cache and the underlying database.

src/calibre/library/save_to_disk.py

@@ -113,8 +113,9 @@ def config(defaults=None):
             help=_('The format in which to display dates. %d - day, %b - month, '
                 '%Y - year. Default is: %b, %Y'))
     x('send_timefmt', default='%b, %Y',
-            help=_('The format in which to display dates. %d - day, %b - month, '
-                '%Y - year. Default is: %b, %Y'))
+            help=_('The format in which to display dates. %(day)s - day,'
+                ' %(month)s - month, %(year)s - year. Default is: %(default)s'
+                )%dict(day='%d', month='%b', year='%Y', default='%b, %Y'))
     x('to_lowercase', default=False,
             help=_('Convert paths to lowercase.'))
     x('replace_whitespace', default=False,

src/calibre/library/server/mobile.py

@@ -153,12 +153,22 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS,
         bookt.append(TR(thumbnail, data))
     # }}}
 
+    body.append(HR())
+    body.append(DIV(
+        A(_('Switch to the full interface (non-mobile interface)'),
+            href="/browse",
+            style="text-decoration: none; color: blue",
+            title=_('The full interface gives you many more features, '
+                'but it may not work well on a small screen')),
+        style="text-align:center"))
     return HTML(
         HEAD(
             TITLE(__appname__ + ' Library'),
             LINK(rel='icon', href='http://calibre-ebook.com/favicon.ico',
                 type='image/x-icon'),
-            LINK(rel='stylesheet', type='text/css', href=prefix+'/mobile/style.css')
+            LINK(rel='stylesheet', type='text/css',
+                href=prefix+'/mobile/style.css'),
+            LINK(rel='apple-touch-icon', href="/static/calibre.png")
         ), # End head
         body
     ) # End html

src/calibre/manual/conf.py

@@ -96,8 +96,8 @@ html_sidebars = {
 html_favicon = 'favicon.ico'
 
 # Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
+# relative to this directory. They are copied after the built-in static files,
+# so a file named "default.css" will overwrite the built-in "default.css".
 html_static_path = ['resources', '../../../icons/favicon.ico']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,

src/calibre/manual/conversion.rst

@@ -2,16 +2,16 @@
 
 .. _conversion:
 
-E-book Conversion
+Ebook Conversion
 ===================
 
 |app| has a conversion system that is designed to be very easy to use. Normally, you just
 add a book to |app|, click convert and |app| will try hard to generate output that is as
 close as possible to the input. However, |app| accepts a very large number of input formats,
-not all of which are as suitable as others for conversion to e-books. In the case of
+not all of which are as suitable as others for conversion to ebooks. In the case of
 such input formats, or if you just want greater control over the conversion system,
 |app| has a lot of options to fine tune the conversion process. Note however that |app|'s
-conversion system is not a substitute for a full blown e-book editor. To edit e-books, I
+conversion system is not a substitute for a full blown ebook editor. To edit ebooks, I
 would recommend first converting them to EPUB using |app| and then using a dedicated EPUB editor,
 like `Sigil <http://code.google.com/p/sigil/>`_ to get the book into perfect shape. You can then
 use the edited EPUB as input for conversion into other formats in |app|.
@@ -23,7 +23,7 @@ mouse over it, a tooltip will appear describing the setting.
 
 .. image:: images/conv_dialog.png
     :align: center
-    :alt: E-book conversion dialog
+    :alt: Ebook conversion dialog
     :scale: 50 
 
 .. contents:: Contents
@@ -75,7 +75,7 @@ The four sub-directories are:
     input         This contains the HTML output by the Input Plugin. Use this to debug the Input Plugin.
     parsed        The result of pre-processing and converting to XHTML the output from the Input Plugin. Use to debug structure detection. 
     structure     Post structure detection, but before CSS flattening and font size conversion. Use to debug font size conversion and CSS transforms.
-    processed     Just before the e-book is passed to the output plugin. Use to debug the Output Plugin.
+    processed     Just before the ebook is passed to the output plugin. Use to debug the Output Plugin.
     ==========    =============
 
 If you want to edit the input document a little before having |app| convert it, the best thing to 
@@ -94,7 +94,7 @@ Look & Feel
   :depth: 1
   :local:
 
-This group of options controls various aspects of the look and feel of the converted e-book. 
+This group of options controls various aspects of the look and feel of the converted ebook. 
 
 .. _font-size-rescaling:
 
@@ -209,7 +209,7 @@ Miscellaneous
 There are a few more options in this section.
 
 :guilabel:`No text justification`
-    Normally, if the output format supports it, |app| will force the output e-book
+    Normally, if the output format supports it, |app| will force the output ebook
     to have *justified* text (i.e., a smooth right margin). This option will turn
     off this behavior, in which case whatever justification is specified in the input document
     will be used instead.
@@ -227,7 +227,7 @@ There are a few more options in this section.
     with "Mikhail Gorbachiov". Also, note that in cases where there are multiple representations
     of a character (characters shared by Chinese and Japanese for instance) the representation used
     by the largest number of people will be used (Chinese in the previous example).
-    This option is mainly useful if you are going to view the e-book on a device that does not
+    This option is mainly useful if you are going to view the ebook on a device that does not
     have support for unicode.
 
 :guilabel:`Input character encoding`
@@ -416,7 +416,7 @@ There are a few more options in this section.
 :guilabel:`Insert metadata as page at start of book`
     One of the great things about |app| is that it allows you to maintain very complete metadata
     about all of your books, for example, a rating, tags, comments, etc. This option will create
-    a single page with all this metadata and insert it into the converted e-book, typically just
+    a single page with all this metadata and insert it into the converted ebook, typically just
     after the cover. Think of it as a way to create your own customised book jacket.
 
 :guilabel:`Remove first image`
@@ -432,7 +432,7 @@ Table of Contents
 When the input document has a Table of Contents in its metadata, |app| will just use that. However,
 a number of older formats either do not support a metadata based Table of Contents, or individual
 documents do not have one. In these cases, the options in this section can help you automatically
-generate a Table of Contents in the converted e-book, based on the actual content in the input document.
+generate a Table of Contents in the converted ebook, based on the actual content in the input document.
 
 The first option is :guilabel:`Force use of auto-generated Table of Contents`. By checking this option
 you can have |app| override any Table of Contents found in the metadata of the input document with the
@@ -681,7 +681,7 @@ The .cbc file will then contain::
     two.cbz
     three.cbz
 
-|app| will automatically convert this .cbc file into a e-book with a Table of Contents pointing to each entry in comics.txt.
+|app| will automatically convert this .cbc file into a ebook with a Table of Contents pointing to each entry in comics.txt.
 
 
 EPUB advanced formatting demo
@@ -695,5 +695,5 @@ EPUB from the ZIP file are::
 
     ebook-convert demo.zip .epub -vv --authors "Kovid Goyal" --language en --level1-toc '//*[@class="title"]' --disable-font-rescaling --page-breaks-before / --no-default-epub-cover
 
-Note that because this file explores the potential of EPUB, most of the advanced formatting is not going to work on readers less capable than |app|'s builtin EPUB viewer. 
+Note that because this file explores the potential of EPUB, most of the advanced formatting is not going to work on readers less capable than |app|'s built-in EPUB viewer. 
 

src/calibre/manual/creating_plugins.rst

@@ -6,7 +6,7 @@
 Writing your own plugins to extend |app|'s functionality
 ====================================================================
 
-|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the builtin plugins in |app| by going to :guilabel:`Preferences->Plugins`.
+|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the built-in plugins in |app| by going to :guilabel:`Preferences->Plugins`.
 
 Here, we will teach you how to create your own plugins to add new features to |app|.
 
@@ -127,7 +127,7 @@ The actual logic to implement the Interface Plugin Demo dialog.
 Getting resources from the plugin zip file
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-|app|'s plugin loading system defines a couple of builtin functions that allow you to conveniently get files from the plugin zip file.
+|app|'s plugin loading system defines a couple of built-in functions that allow you to conveniently get files from the plugin zip file.
 
     **get_resources(name_or_list_of_names)**
         This function should be called with a list of paths to files inside the zip file. For example to access the file icon.png in

src/calibre/manual/customize.rst

@@ -53,15 +53,15 @@ from the calibre website it will be :file:`/opt/calibre/resources`. These paths
 
 You should not change the files in this resources folder, as your changes will get overwritten the next time you update |app|. Instead, go to
 :guilabel:`Preferences->Advanced->Miscellaneous` and click :guilabel:`Open calibre configuration directory`. In this configuration directory, create a sub-folder called resources and place the files you want to override in it. Place the files in the appropriate sub folders, for example place images in :file:`resources/images`, etc. 
-|app| will automatically use your custom file in preference to the builtin one the next time it is started.
+|app| will automatically use your custom file in preference to the built-in one the next time it is started.
 
-For example, if you wanted to change the icon for the :guilabel:`Remove books` action, you would first look in the builtin resources folder and see that the relevant file is
+For example, if you wanted to change the icon for the :guilabel:`Remove books` action, you would first look in the built-in resources folder and see that the relevant file is
 :file:`resources/images/trash.png`. Assuming you have an alternate icon in PNG format called :file:`mytrash.png` you would save it in the configuration directory as :file:`resources/images/trash.png`. All the icons used by the calibre user interface are in :file:`resources/images` and its sub-folders.
 
 Customizing |app| with plugins
 --------------------------------
 
-|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the builtin plugins in |app| by going to :guilabel:`Preferences->Plugins`.
+|app| has a very modular design. Almost all functionality in |app| comes in the form of plugins. Plugins are used for conversion, for downloading news (though these are called recipes), for various components of the user interface, to connect to different devices, to process files when adding them to |app| and so on. You can get a complete list of all the built-in plugins in |app| by going to :guilabel:`Preferences->Plugins`.
 
 You can write your own plugins to customize and extend the behavior of |app|. The plugin architecture in |app| is very simple, see the tutorial :ref:`pluginstutorial`.
 

src/calibre/manual/develop.rst

@@ -6,7 +6,7 @@ Setting up a |app| development environment
 ===========================================
 
 |app| is completely open source, licensed under the `GNU GPL v3 <http://www.gnu.org/copyleft/gpl.html>`_.
-This means that you are free to download and modify the program to your hearts content. In this section, 
+This means that you are free to download and modify the program to your heart's content. In this section, 
 you will learn how to get a |app| development environment set up on the operating system of your choice. 
 |app| is written primarily in `Python <http://www.python.org>`_ with some C/C++ code for speed and system interfacing. 
 Note that |app| is not compatible with Python 3 and requires at least Python 2.7.
@@ -18,7 +18,7 @@ Note that |app| is not compatible with Python 3 and requires at least Python 2.7
 Design philosophy
 -------------------
 
-|app| has its roots in the Unix world, which means that it's design is highly modular.
+|app| has its roots in the Unix world, which means that its design is highly modular.
 The modules interact with each other via well defined interfaces. This makes adding new features and fixing
 bugs in |app| very easy, resulting in a frenetic pace of development. Because of its roots, |app| has a 
 comprehensive command line interface for all its functions, documented in :ref:`cli`.
@@ -28,7 +28,7 @@ For example, adding support for a new device to |app| typically involves writing
 a device driver plugin. You can browse the 
 `built-in drivers <http://bazaar.launchpad.net/%7Ekovid/calibre/trunk/files/head%3A/src/calibre/devices/>`_. Similarly, adding support 
 for new conversion formats involves writing input/output format plugins. Another example of the modular design is the :ref:`recipe system <news>` for 
-fetching news. For more examples of plugins designed to add features to |app|, see the `plugin index <http://www.mobileread.com/forums/showthread.php?t=118764>`_.
+fetching news. For more examples of plugins designed to add features to |app|, see the `plugin index <http://www.mobileread.com/forums/showthread.php?p=1362767#post1362767>`_.
 
 Code layout
 ^^^^^^^^^^^^^^
@@ -38,15 +38,15 @@ All the |app| python code is in the ``calibre`` package. This package contains t
     * devices - All the device drivers. Just look through some of the built-in drivers to get an idea for how they work.
 
       * For details, see: devices.interface which defines the interface supported by device drivers and devices.usbms which
-        defines a generic driver that connects to a USBMS device. All USBMS based drivers in calibre inherit from it.
+        defines a generic driver that connects to a USBMS device. All USBMS based drivers in |app| inherit from it.
 
     * ebooks  - All the ebook conversion/metadata code. A good starting point is ``calibre.ebooks.conversion.cli`` which is the
       module powering the :command:`ebook-convert` command. The conversion process is controlled via conversion.plumber.
-      The format independent code is all in ebooks.oeb and the format dependent stuff is in ebooks.format_name.
+      The format independent code is all in ebooks.oeb and the format dependent code is in ebooks.format_name.
 
-        * Metadata reading writing and downloading is all in ebooks.metadata
+        * Metadata reading, writing, and downloading is all in ebooks.metadata
 
-    * library - The database backed and the content server. See library.database2 for the interface to the calibre library. library.server is the calibre Content Server.
+    * library - The database back-end and the content server. See library.database2 for the interface to the |app| library. library.server is the |app| Content Server.
     * gui2 - The Graphical User Interface. GUI initialization happens in gui2.main and gui2.ui. The ebook-viewer is in gui2.viewer.
 
 If you need help understanding the code, post in the `development forum <http://www.mobileread.com/forums/forumdisplay.php?f=240>`_
@@ -55,8 +55,8 @@ and you will most likely get help from one of |app|'s many developers.
 Getting the code
 ------------------
 
-|app| uses `Bazaar <http://bazaar-vcs.org/>`_ a distributed version control system. Bazaar is available on all the platforms |app| supports.
-After installing Bazaar, you can get the calibre source code with the command::
+|app| uses `Bazaar <http://bazaar-vcs.org/>`_, a distributed version control system. Bazaar is available on all the platforms |app| supports.
+After installing Bazaar, you can get the |app| source code with the command::
 
     bzr branch lp:calibre
 
@@ -68,7 +68,7 @@ to the latest code, use the command::
 Submitting your changes to be included
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you only plan to only make a few small changes, you can make your changes and create a
+If you only plan to make a few small changes, you can make your changes and create a
 "merge directive" which you can then attach to a ticket in the |app| bug tracker for consideration. To do
 this, make your changes, then run::
 
@@ -76,11 +76,11 @@ this, make your changes, then run::
     bzr send -o my-changes
 
 This will create a :file:`my-changes` file in the current directory,
-simply attach that to a ticket on the |app| `bug tracker <http://calibre-ebook.com/bugs>`_.
+simply attach that to a ticket on the |app| `bug tracker <https://bugs.launchpad.net/calibre>`_.
 
 If you plan to do a lot of development on |app|, then the best method is to create a 
-`Launchpad <http://launchpad.net>`_ account. Once you have the account, you can use it to register
-your bzr branch created by the `bzr branch` command above with the |app| project. First run the
+`Launchpad <http://launchpad.net>`_ account. Once you have an account, you can use it to register
+your bzr branch created by the `bzr branch` command above. First run the
 following command to tell bzr about your launchpad account::
 
     bzr launchpad-login your_launchpad_username
@@ -88,31 +88,31 @@ following command to tell bzr about your launchpad account::
 Now, you have to setup SSH access to Launchpad. First create an SSH public/private keypair. Then upload 
 the public key to Launchpad by going to your Launchpad account page. Instructions for setting up the 
 private key in bzr are at http://bazaar-vcs.org/Bzr_and_SSH. Now you can upload your branch to the |app|
-project in Launchapd by following the instructions at https://help.launchpad.net/Code/UploadingABranch.
-Now whenever you commit changes to your branch with the command::
+project in Launchpad by following the instructions at https://help.launchpad.net/Code/UploadingABranch.
+Whenever you commit changes to your branch with the command::
 
     bzr commit -m "Comment describing your change"
 
-I can merge it directly from you branch into the main |app| source tree. You should also keep an eye on the |app|
+Kovid can merge it directly from your branch into the main |app| source tree. You should also keep an eye on the |app|
 `development forum <http://www.mobileread.com/forums/forumdisplay.php?f=240>`. Before making major changes, you should
 discuss them in the forum or contact Kovid directly (his email address is all over the source code). 
 
 Windows development environment
 ---------------------------------
 
-Install |app| normally, using the windows installer. Then, open a Command Prompt and change to
-the previously checked out calibre code directory, for example::
+Install |app| normally, using the Windows installer. Then open a Command Prompt and change to
+the previously checked out |app| code directory. For example::
 
     cd C:\Users\kovid\work\calibre
 
-calibre is the directory that contains the src and resources sub directories. 
+calibre is the directory that contains the src and resources sub-directories. 
 
-The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path to the src directory.
-So, following the example above, it would be ``C:\Users\kovid\work\calibre\src``. A short
-`guide <http://docs.python.org/using/windows.html#excursus-setting-environment-variables>`_ to setting environment
-variables on windows. 
+The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path of the src directory.
+So, following the example above, it would be ``C:\Users\kovid\work\calibre\src``. `Here is a short
+guide <http://docs.python.org/using/windows.html#excursus-setting-environment-variables>`_ to setting environment
+variables on Windows. 
 
-Once you have set the environment variable, open a new Command Prompt and check that it was correctly set by using
+Once you have set the environment variable, open a new command prompt and check that it was correctly set by using
 the command::
 
     echo %CALIBRE_DEVELOP_FROM%
@@ -129,14 +129,14 @@ near the top of the file. Now run the command :command:`calibredb`. The very fir
 OS X development environment
 ------------------------------
 
-Install |app| normally, using the provided .dmg. Then, open a Terminal and change to
-the previously checked out calibre code directory, for example::
+Install |app| normally using the provided .dmg. Then open a Terminal and change to
+the previously checked out |app| code directory, for example::
 
     cd /Users/kovid/work/calibre
 
-calibre is the directory that contains the src and resources sub directories. Ensure you have installed the |app| commandline tools via :guilabel:Preferences->Advanced->Miscellaneous in the |app| GUI.
+calibre is the directory that contains the src and resources sub-directories. Ensure you have installed the |app| commandline tools via :guilabel:Preferences->Advanced->Miscellaneous in the |app| GUI.
 
-The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path to the src directory.
+The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path of the src directory.
 So, following the example above, it would be ``/Users/kovid/work/calibre/src``. Apple 
 `documentation <http://developer.apple.com/mac/library/documentation/MacOSX/Conceptual/BPRuntimeConfig/Articles/EnvironmentVars.html#//apple_ref/doc/uid/20002093-BCIJIJBH>`_
 on how to set environment variables. 
@@ -158,21 +158,21 @@ near the top of the file. Now run the command :command:`calibredb`. The very fir
 Linux development environment
 ------------------------------
 
-|app| is primarily developed on linux. You have two choices in setting up the development environment. You can install the
+|app| is primarily developed on Linux. You have two choices in setting up the development environment. You can install the
 |app| binary as normal and use that as a runtime environment to do your development. This approach is similar to that
-used in windows and OS X. Alternatively, you can install |app| from source. Instructions for setting up a development
-environment from source are in the INSTALL file in the source tree. Here we will address using the binary a runtime, which is the
+used in Windows and OS X. Alternatively, you can install |app| from source. Instructions for setting up a development
+environment from source are in the INSTALL file in the source tree. Here we will address using the binary at runtime, which is the
 recommended method.
 
 Install the |app| using the binary installer. Then open a terminal and change to the previously checked out |app| code directory, for example::
 
     cd /home/kovid/work/calibre
 
-calibre is the directory that contains the src and resources sub directories. 
+calibre is the directory that contains the src and resources sub-directories. 
 
-The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path to the src directory.
+The next step is to set the environment variable ``CALIBRE_DEVELOP_FROM`` to the absolute path of the src directory.
 So, following the example above, it would be ``/home/kovid/work/calibre/src``. How to set environment variables depends on
-your linux distribution and what shell you are using. 
+your Linux distribution and what shell you are using. 
 
 Once you have set the environment variable, open a new terminal and check that it was correctly set by using
 the command::
@@ -191,10 +191,10 @@ near the top of the file. Now run the command :command:`calibredb`. The very fir
 Having separate "normal" and "development" |app| installs on the same computer
 -------------------------------------------------------------------------------
 
-The calibre source tree is very stable, it rarely breaks, but if you feel the need to run from source on a separate
-test library and run the released calibre version with your everyday library, you can achieve this easily using
-.bat files or shell scripts to launch |app|. The example below shows how to do this on windows using .bat files (the
-instructions for other platforms are the same, just use a BASH script instead of a .bat file)
+The |app| source tree is very stable and rarely breaks, but if you feel the need to run from source on a separate
+test library and run the released |app| version with your everyday library, you can achieve this easily using
+.bat files or shell scripts to launch |app|. The example below shows how to do this on Windows using .bat files (the
+instructions for other platforms are the same, just use a shell script instead of a .bat file)
 
 To launch the release version of |app| with your everyday library:
 
@@ -211,9 +211,9 @@ calibre-dev.bat::
 Debugging tips
 ----------------
 
-Running calibre code in a python debugger is not easy, unless you install from source on linux. However, python is a
-dynamically typed language with excellent facilities for introspection. I wrote the core calibre code without once
-using a debugger. There are two main strategies to debug calibre code:
+Python is a
+dynamically typed language with excellent facilities for introspection. Kovid wrote the core |app| code without once
+using a debugger. There are many strategies to debug |app| code:
 
 Using an interactive python interpreter
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -223,15 +223,15 @@ You can insert the following two lines of code to start an interactive python se
     from calibre import ipython
     ipython(locals())
 
-When running from the command line, this will start an interactive python interpreter with access to all
+When running from the command line, this will start an interactive Python interpreter with access to all
 locally defined variables (variables in the local scope). The interactive prompt even has TAB completion
-for object properties and you can use the various python facilities for introspection, such as
+for object properties and you can use the various Python facilities for introspection, such as
 :func:`dir`, :func:`type`, :func:`repr`, etc.
 
 Using print statements
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-This is my favorite way to debug. Simply insert print statements at points of interest and run your program in the
+This is Kovid's favorite way to debug. Simply insert print statements at points of interest and run your program in the
 terminal. For example, you can start the GUI from the terminal as::
 
     calibre-debug -g
@@ -240,11 +240,17 @@ Similarly, you can start the ebook-viewer as::
 
     calibre-debug -w /path/to/file/to/be/viewed
 
-Executing arbitrary scripts in the calibre python environment
+Using the debugger in PyDev
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+It is possible to get the debugger in PyDev working with the |app| development environment,
+see the `forum thread <http://www.mobileread.com/forums/showthread.php?t=143208>`_.
+
+Executing arbitrary scripts in the |app| python environment
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 The :command:`calibre-debug` command provides a couple of handy switches to execute your own
-code, with access to the calibre modules::
+code, with access to the |app| modules::
 
     calibre-debug -c "some python code"
 
@@ -252,33 +258,33 @@ is great for testing a little snippet of code on the command line. It works in t
 
     calibre-debug -e myscript.py
 
-can be used to execute your own python script. It works in the same way as passing the script to the python interpreter, except
+can be used to execute your own Python script. It works in the same way as passing the script to the Python interpreter, except
 that the calibre environment is fully initialized, so you can use all the calibre code in your script.
 
 
-Using calibre in your projects
+Using |app| in your projects
 ----------------------------------------
 
-It is possible to directly use calibre functions/code in your python project. Two ways exist to do this:
+It is possible to directly use |app| functions/code in your Python project. Two ways exist to do this:
 
-Binary install of calibre
+Binary install of |app|
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you have a binary install of calibre, you can use the python interpreter bundled with calibre, like this::
+If you have a binary install of |app|, you can use the Python interpreter bundled with |app|, like this::
 
     calibre-debug -e /path/to/your/python/script.py
 
-Source install on linux
+Source install on Linux
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-In addition to using the above technique, if you do a source install on linux,
-you can also directly import calibre, as follows::
+In addition to using the above technique, if you do a source install on Linux,
+you can also directly import |app|, as follows::
 
     import init_calibre
     import calibre
 
     print calibre.__version__
 
-It is essential that you import the init_calibre module before any other calibre modules/packages as
-it sets up the interpreter to run calibre code.
+It is essential that you import the init_calibre module before any other |app| modules/packages as
+it sets up the interpreter to run |app| code.
 

src/calibre/manual/faq.rst

@@ -9,7 +9,7 @@ Frequently Asked Questions
   :depth: 1
   :local:
 
-E-book Format Conversion
+Ebook Format Conversion
 -------------------------
 .. contents:: Contents
   :depth: 1
@@ -61,7 +61,7 @@ In the MOBI format, the situation is a little confused. This is because the MOBI
 
 Now it might well seem to you that the MOBI book has two identical TOCs. Remember that one is semantically a content TOC and the other is a metadata TOC, even though both might have exactly the same entries and look the same. One can be accessed directly from the Kindle's menus, the other cannot. 
 
-When converting to MOBI, calibre detects the *metadata TOC* in the input document and generates an end-of-file TOC in the output MOBI file. You can turn this off by an option in the MOBI Output settings. You cannot control where this generated TOC will go. Remember this TOC is semantically a *metadata TOC*, in any format other than MOBI it *cannot not be part of the text*. The fact that it is part of the text in MOBI is an accident caused by the limitations of MOBI. If you want a TOC at a particular location in your document text, create one by hand. 
+When converting to MOBI, calibre detects the *metadata TOC* in the input document and generates an end-of-file TOC in the output MOBI file. You can turn this off by an option in the MOBI Output settings. You can also tell calibre whether to put it and the start or the end of the book via an option in the MOBI Output settings. Remember this TOC is semantically a *metadata TOC*, in any format other than MOBI it *cannot not be part of the text*. The fact that it is part of the text in MOBI is an accident caused by the limitations of MOBI. If you want a TOC at a particular location in your document text, create one by hand. So we strongly recommend that you leave the default as it is, i.e. with the metadata TOC at the end of the book.
 
 If you have a hand edited TOC in the input document, you can use the TOC detection options in calibre to automatically generate the metadata TOC from it. See the conversion section of the User Manual for more details on how to use these options.
 
@@ -112,8 +112,8 @@ How can I help get my device supported in |app|?
 If your device appears as a USB disk to the operating system, adding support for it to |app| is very easy.
 We just need some information from you:
 
-  * What e-book formats does your device support?
-  * Is there a special directory on the device in which all e-book files should be placed?
+  * What ebook formats does your device support?
+  * Is there a special directory on the device in which all ebook files should be placed?
   * We also need information about your device that |app| will collect automatically. First, if your
     device supports SD cards, insert them. Then connect your device. In calibre go to :guilabel:`Preferences->Advanced->Miscellaneous`
     and click the "Debug device detection" button. This will create some debug output. Copy it to a file
@@ -259,7 +259,7 @@ This method only works on Windows XP and higher, and OS X 10.5 and higher. Linux
 How do I use |app| with my Android phone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-First install the WordPlayer e-book reading app from the Android Marketplace onto you phone. Then simply plug your phone into the computer with a USB cable. |app| should automatically detect the phone and then you can transfer books to it by clicking the Send to Device button. |app| does not have support for every single androind device out there, so if you would like to have support for your device added, follow the instructions above for getting your device supported in |app|.
+First install the WordPlayer ebook reading app from the Android Marketplace onto you phone. Then simply plug your phone into the computer with a USB cable. |app| should automatically detect the phone and then you can transfer books to it by clicking the Send to Device button. |app| does not have support for every single androind device out there, so if you would like to have support for your device added, follow the instructions above for getting your device supported in |app|.
 
 Can I access my |app| books using the web browser in my Kindle or other reading device?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -281,6 +281,15 @@ I get the error message "Failed to start content server: Port 8080 not free on '
 
 The most likely cause of this is your antivirus program. Try temporarily disabling it and see if it does the trick.
 
+I cannot send emails using |app|?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Because of the large amount of spam in email, sending email can be tricky as different servers use different strategies to block email spam. 
+The most common problem is if you are sending email directly (without a mail relay) in |app|. Many servers (for example, Amazon) block email
+that does not come from a well known relay. The easiest way around this is to setup a free GMail account and then goto Preferences->Email in |app| and
+click the "Use Gmail" button. |app| will then use Gmail to send the mail. Remember to update the email preferences in on your Amazon Kindle page to
+allow email sent from your Gmail email address.
+
 Why is my device not detected in linux?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -382,7 +391,7 @@ With all this flexibility, it is possible to have |app| manage your author names
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The whole point of |app|'s library management features is that they provide a search and sort based interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization. To illustrate why a search/tagging based interface is superior to folders, consider the following. Suppose your book collection is nicely sorted into folders with the following scheme::
+The whole point of |app|'s library management features is that they provide a search and sort based interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory structure of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization. To illustrate why a search/tagging based interface is superior to folders, consider the following. Suppose your book collection is nicely sorted into folders with the following scheme::
 
     Genre -> Author -> Series -> ReadStatus
 
@@ -392,6 +401,14 @@ Now this makes it very easy to find for example all science fiction books by Isa
 
 In |app|, you would instead use tags to mark genre and read status and then just use a simple search query like ``tag:scifi and not tag:read``. |app| even has a nice graphical interface, so you don't need to learn its search language instead you can just click on tags to include or exclude them from the search.
 
+To those of you that claim that you need access to the filesystem to so that you can have access to your books over the network, |app| has an excellent content server that gives you access to your calibre library over the net.
+
+If you are worried that someday |app| will cease to be developed, leaving all your books marooned in its folder structure, explore the powerful "Save to Disk" feature in |app| that lets you export all your files into a folder structure of arbitrary complexity based on their metadata.
+
+Finally, the reason there are numbers at the end of every title folder, is for *robustness*. That number is the id number of the book record in the |app| database. The presence of the number allows you to have multiple records with the same title and author names. It is also part of what allows |app| to magically regenerate the database with all metadata if the database file gets corrupted. Given that |app|'s mission is to get you to stop storing metadata in filenames and stop using the filesystem to find things, the increased robustness afforded by the id numbers is well worth the uglier folder names.
+
+If you are still not convinced, then I'm afraid |app| is not for you. Look elsewhere for your book cataloguing needs. Just so we're clear, **this is not going to change**. Kindly do not contact us in an attempt to get us to change this.
+
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 |app| is designed to have columns for the most frequently and widely used fields. In addition, you can add any columns you like. Columns can be added via :guilabel:`Preferences->Interface->Add your own columns`.
@@ -408,8 +425,9 @@ How do I move my |app| library from one computer to another?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Simply copy the |app| library folder from the old to the new computer. You can find out what the library folder is by clicking the calibre icon in the toolbar. The very first item is the path to the library folder. Now on the new computer, start |app| for the first time. It will run the Welcome Wizard asking you for the location of the |app| library. Point it to the previously copied folder. If the computer you are transferring to already has a calibre installation, then the Welcome wizard wont run. In that case, click the calibre icon in the tooolbar and point it to the newly copied directory. You will now have two calibre libraries on your computer and you can switch between them by clicking the calibre icon on the toolbar.
 
-Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also go to :guilabel:`Preferences->Advanced->Miscellaneous` and click the "Check database integrity button". It will warn you about missing files, if any, which you should then transfer by hand.
+Note that if you are transferring between different types of computers (for example Windows to OS X) then after doing the above you should also click the arrow next to the calibre icon on the tool bar, select Library Maintenance and run the Check Library action. It will warn you about any problems in your library, which you should fix by hand.
 
+.. note:: A |app| library is just a folder which contains all the book files and their metadata. All the emtadata is stored in a single file called metadata.db, in the top level folder. If this file gets corrupted, you may see an empty list of books in |app|. In this case you can ask |app| to restore your books by clicking the arrow next to the |app| icon on the toolbar and selecting Library Maintenance->Restore Library.
 
 Content From The Web
 ---------------------
@@ -442,16 +460,16 @@ Miscellaneous
 Why the name calibre?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Take your pick:
-  * Convertor And LIBRary for E-books
+  * Convertor And LIBRary for Ebooks
   * A high *calibre* product
-  * A tribute to the SONY Librie which was the first e-ink based e-book reader
+  * A tribute to the SONY Librie which was the first e-ink based ebook reader
   * My wife chose it ;-)
 
 |app| is pronounced as cal-i-ber *not* ca-li-bre. If you're wondering, |app| is the British/commonwealth spelling for caliber. Being Indian, that's the natural spelling for me. 
 
 Why does |app| show only some of my fonts on OS X?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-|app| embeds fonts in ebook files it creates. E-book files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts found on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
+|app| embeds fonts in ebook files it creates. Ebook files support embedding only TrueType (.ttf) fonts. Most fonts on OS X systems are in .dfont format, thus they cannot be embedded. |app| shows only TrueType fonts found on your system. You can obtain many TrueType fonts on the web. Simply download the .ttf files and add them to the Library/Fonts directory in your home directory.
 
 |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -462,7 +480,7 @@ There can be several causes for this:
        * Uninstall calibre
        * Reboot your computer
        * Re-install calibre. But do not start calibre from the installation wizard.
-       * Temporarily disable your antivirus program (disconnect from the internet before doing so, to be safe)
+       * Temporarily disable your antivirus program (disconnect from the Internet before doing so, to be safe)
        * Look inside the folder you chose for your calibre library. If you see a file named metadata.db, delete it.
        * Start calibre
        * From now on you should be able to start calibre normally.
@@ -515,7 +533,7 @@ menu, choose "Validate fonts".
 I downloaded the installer, but it is not working?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Downloading from the internet can sometimes result in a corrupted download. If the |app| installer you downloaded is not opening, try downloading it again. If re-downloading it does not work, download it from `an alternate location <http://sourceforge.net/projects/calibre/files/>`_. If the installer still doesn't work, then something on your computer is preventing it from running. 
+Downloading from the Internet can sometimes result in a corrupted download. If the |app| installer you downloaded is not opening, try downloading it again. If re-downloading it does not work, download it from `an alternate location <http://sourceforge.net/projects/calibre/files/>`_. If the installer still doesn't work, then something on your computer is preventing it from running. 
 
     * Try temporarily disabling your antivirus program (Microsoft Security Essentials, or Kaspersky or Norton or McAfee or whatever). This is most likely the culprit if the upgrade process is hanging in the middle.
     * Try rebooting your computer and running a registry cleaner like `Wise registry cleaner <http://www.wisecleaner.com>`_.
@@ -539,7 +557,7 @@ If you want to backup the |app| configuration/plugins, you have to backup the co
 
 How do I use purchased EPUB books with |app|?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Most purchased EPUB books have `DRM <http://wiki.mobileread.com/wiki/DRM>`_. This prevents |app| from opening them. You can still use |app| to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with |app| will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to |app|.
+Most purchased EPUB books have `DRM <http://drmfree.calibre-ebook.com/about#drm>`_. This prevents |app| from opening them. You can still use |app| to store and transfer them to your ebook reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with |app| will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" ebook. The ebook file will be stored in the folder "My Digital Editions", from where you can add it to |app|.
 
 I am getting a "Permission Denied" error?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/glossary.rst

@@ -6,25 +6,25 @@ Glossary
 .. glossary::
 
     RSS
-       **RSS** *(Really Simple Syndication)* is a web feed format that is used to publish frequently updated content, like news articles, blog posts, etc. It is a format that is particularly suited to being read by computers, and is therefore the preferred way of getting content from the web into an e-book. There are many other feed formats in use on the internet, and |app| understands most of them. In particular, it has good support for the *ATOM* format, which is commonly used for blogs. 
+       **RSS** *(Really Simple Syndication)* is a web feed format that is used to publish frequently updated content, like news articles, blog posts, etc. It is a format that is particularly suited to being read by computers, and is therefore the preferred way of getting content from the web into an ebook. There are many other feed formats in use on the Internet, and |app| understands most of them. In particular, it has good support for the *ATOM* format, which is commonly used for blogs. 
 
     recipe
-        A recipe is a set of instructions that teach |app| how to convert an online news source, like a magazine or a blog into an e-book. A recipe, is essentially `python <http://www.python.org>`_ code. As such, it is capable of converting arbitrarily complex news sources into e-books. At the simplest level, it is just a set of variables such as URLs that give |app| enough information to go out onto the internet and download the news. 
+        A recipe is a set of instructions that teach |app| how to convert an online news source, such as a magazine or a blog, into an ebook. A recipe is essentially `Python <http://www.python.org>`_ code. As such, it is capable of converting arbitrarily complex news sources into ebooks. At the simplest level, it is just a set of variables, such as URLs, that give |app| enough information to go out onto the Internet and download the news. 
 
     HTML
-        **HTML** *(Hyper Text Mark-Up Language)*, a subset of Standard Generalized Mark-Up Language (SGML) for electronic publishing, the specific standard used for the World Wide Web.
+        **HTML** *(Hyper Text Mark-Up Language)*, a subset of Standard Generalized Mark-Up Language (SGML) for electronic publishing, is the specific standard used for the World Wide Web.
 
     CSS
-        **CSS** *(Cascading Style Sheets)* a language used to describe how an :term:`HTML` document should be rendered (visual styling).
+        **CSS** *(Cascading Style Sheets)* is a language used to describe how an :term:`HTML` document should be rendered (visual styling).
 
     API
         **API** *(Application Programming Interface)* is a source code interface that a library provides to support requests for services to be made of it by computer programs.
 
     LRF
-        **LRF** The e-book format that is read by the SONY e-book readers.
+        **LRF** The ebook format that is read by the SONY ebook readers.
 
     URL
        **URL** *(Uniform Resource Locator)* for example: ``http://example.com``
 
     regexp
-         **Regular expressions** provide a concise and flexible means for identifying strings of text of interest, such as particular characters, words, or patterns of characters. See `regexp syntax <http://docs.python.org/lib/re-syntax.html>`_ for the syntax of regular expressions used in python.
+         **Regular expressions** provide a concise and flexible means for identifying strings of text of interest, such as particular characters, words, or patterns of characters. See `regexp syntax <http://docs.python.org/lib/re-syntax.html>`_ for the syntax of regular expressions used in Python.

src/calibre/manual/gui.rst

@@ -9,10 +9,11 @@ The Graphical User Interface *(GUI)* provides access to all
 library management and ebook format conversion features. The basic workflow
 for using |app| is to first add books to the library from your hard disk.
 |app| will automatically try to read metadata from the books and add them
-to its internal database. Once they are in the database, you can perform a various
+to its internal database. Once they are in the database, you can perform various
 :ref:`actions` on them that include conversion from one format to another,
-transfer to the reading device, viewing on your computer, editing metadata, including covers, etc.
-Note that |app| creates copies of the files you add to it, your original files are left untouched.
+transfer to the reading device, viewing on your computer, and editing metadata.
+The latter includes modifying the cover, description, and tags among other details.
+Note that |app| creates copies of the files you add to it. Your original files are left untouched.
 
 The interface is divided into various sections:
 
@@ -42,25 +43,25 @@ Add books
 .. |adbi| image:: images/add_books.png
     :class: float-right-img
 
-|adbi| The :guilabel:`Add books` action has five variations, accessed by the clicking the down arrow on the right side of the button.
+|adbi| The :guilabel:`Add books` action has six variations accessed by the clicking the down arrow on the right side of the button.
 
 
     1. **Add books from a single directory**: Opens a file chooser dialog and allows you to specify which books in a directory should be added. This action is *context sensitive*, i.e. it depends on which :ref:`catalog <catalogs>` you have selected. If you have selected the :guilabel:`Library`, books will be added to the library. If you have selected the ebook reader device, the books will be uploaded to the device, and so on.
 
-    2. **Add books from directories, including sub-directories (One book per directory, assumes every ebook file is the same book in a different format)**: Allows you to choose a directory. The directory and all its sub-directories are scanned recursively and any ebooks found are added to the library. The algorithm assumes that each directory contains a single book. All ebook files in a directory are assumedto be the same book in different formats. This action is the inverse of the :ref:`Save to disk <save_to_disk_multiple>` action, i.e. you can :guilabel:`Save to disk`, delete the books and re-add them with no lost information (except date).
+    2. **Add books from directories, including sub-directories (One book per directory, assumes every ebook file is the same book in a different format)**: Allows you to choose a directory. The directory and all its sub-directories are scanned recursively, and any ebooks found are added to the library. |app| assumes that each directory contains a single book. All ebook files in a directory are assumed to be the same book in different formats. This action is the inverse of the :ref:`Save to disk <save_to_disk_multiple>` action, i.e. you can :guilabel:`Save to disk`, delete the books and re-add them with no lost information except for the date.
 
-    3. **Add books directories, including sub-directories (Multiple books per directory, assumes every ebook file is a different book)**: Allows you to choose a directory. The directory and all its sub-directories are scanned recursively and any ebooks found are added to the library. The algorithm assumes that each directory contains many books. All ebook files with the same name in a directory are assumed to be the same book in different formats. Ebooks with different names are added as different books. This action is the inverse of the :ref:`Save to disk <save_to_disk_single>` action, i.e. you can :guilabel:`Save to disk`, delete the books and re-add them with no lost information (except date).
+    3. **Add books from directories, including sub-directories (Multiple books per directory, assumes every ebook file is a different book)**: Allows you to choose a directory. The directory and all its sub-directories are scanned recursively and any ebooks found are added to the library. |app| assumes that each directory contains many books. All ebook files with the same name in a directory are assumed to be the same book in different formats. Ebooks with different names are added as different books. This action is the inverse of the :ref:`Save to disk <save_to_disk_single>` action, i.e. you can :guilabel:`Save to disk`, delete the books and re-add them with no lost information except for the date.
 
-    4. **Add empty book. (Book Entry with blank formats)**: Allows you to create a blank book record. This can be used to then manually fill out the information about a book that you may not have yet in your collection.
+    4. **Add empty book. (Book Entry with no formats)**: Allows you to create a blank book record. This can be used to then manually fill out the information about a book that you may not have yet in your collection.
 
-    5. **Add by ISBN**: Allows you to add one or more books by entering just their ISBN into a list or pasting the list of ISBNs from your clipboard.
+    5. **Add from ISBN**: Allows you to add one or more books by entering their ISBNs.
 
     6. **Add files to selected book records**: Allows you to add or update the files associated with an existing book in your library.
 
-The :guilabel:`Add books` action can read metadata from a wide variety of e-book formats. In addition it tries to guess metadata from the filename.
+The :guilabel:`Add books` action can read metadata from a wide variety of ebook formats. In addition, it tries to guess metadata from the filename.
 See the :ref:`config_filename_metadata` section, to learn how to configure this.
 
-To add an additional format for an existing book, use the :ref:`edit_meta_information` action.
+To add an additional format for an existing book use the :ref:`edit_meta_information` action.
 
 .. _edit_meta_information:
 
@@ -69,41 +70,42 @@ Edit metadata
 .. |emii| image:: images/edit_meta_information.png
     :class: float-right-img
 
-|emii| The :guilabel:`Edit metadata` action has six variations, which can be accessed by clicking the down arrow on the right side of the button.
+|emii| The :guilabel:`Edit metadata` action has four variations which can be accessed by clicking the down arrow on the right side of the button.
 
-    1. **Edit metadata individually**: This allows you to edit the metadata of books one-by-one, with the option of fetching metadata, including covers from the internet. It also allows you to add/remove particular ebook formats from a book. 
-    2. **Edit metadata in bulk**: This allows you to edit common metadata fields for large numbers of books simulataneously. It operates on all the books you have selected in the :ref:`Library view <search_sort>`.
-    3. **Download metadata and covers**: Downloads metadata and covers (if available), for the books that are selected in the book list.
-    4. **Merge Book Records**: Gives you the capability of merging the metadata and formats of two or more book records together. You can choose to either delete or keep the records that were not clicked first.
+    1. **Edit metadata individually**: Allows you to edit the metadata of books one-by-one with the option of fetching metadata, including covers, from the Internet. It also allows you to add or remove particular ebook formats from a book. 
+    2. **Edit metadata in bulk**: Allows you to edit common metadata fields for large numbers of books simulataneously. It operates on all the books you have selected in the :ref:`Library view <search_sort>`.
+    3. **Download metadata and covers**: Downloads metadata and covers (if available) for the books that are selected in the book list.
+    4. **Merge book records**: Gives you the capability of merging the metadata and formats of two or more book records. You can choose to either delete or keep the records that were not clicked first.
 
 For more details see :ref:`metadata`.
 
 .. _convert_ebooks:
 
-Convert e-books
+Convert books
 ~~~~~~~~~~~~~~~~~~~~~~
 .. |cei| image:: images/convert_ebooks.png
     :class: float-right-img
 
-|cei| Ebooks can be converted from a number of formats into whatever format your e-book reader prefers.
-Note that ebooks you purchase will typically have `Digital Rights Management <http://bugs.calibre-ebook.com/wiki/DRM>`_ *(DRM)*.
-|app| will not convert these ebooks. For many DRM formats, it is easy to remove the DRM, but as this may be illegal,
-you have to find tools to liberate your books yourself and then use |app| to convert them.
+|cei| Ebooks can be converted from a number of formats into whatever format your ebook reader prefers.
+Many ebooks available for purchase will be protected by `Digital Rights Management <http://drmfree.calibre-ebook.com/about#drm>`_ *(DRM)* technology.
+|app| will not convert these ebooks. It is easy to remove the DRM from many formats, but as this may be illegal,
+you will have to find tools to liberate your books yourself and then use |app| to convert them.
 
-For most people, conversion should be a simple 1-click affair. But if you want to learn more about the conversion process, see :ref:`conversion`.
+For most people, conversion should be a simple one-click affair. If you want to learn more about the conversion process, see :ref:`conversion`.
 
-The :guilabel:`Convert E-books` action has three variations, accessed by the arrow next to the button.
+The :guilabel:`Convert books` action has three variations, accessed by the arrow next to the button.
 
-    1. **Convert individually**: This will allow you to specify conversion options to customize the conversion of each selected ebook.
+    1. **Convert individually**: Allows you to specify conversion options to customize the conversion of each selected ebook.
 
-    2. **Bulk convert**: This allows you to specify options only once to convert a number of ebooks in bulk.
+    2. **Bulk convert**: Allows you to specify options only once to convert a number of ebooks in bulk.
 
-    3. **Create catalog**: This action allows you to generate a complete listing with all metadata of the books in your library,
-       in several formats, like XML, CSV, BiBTeX, EPUB and MOBI. The catalog will contain all the books showing in the library view currently,
-       so you can use the search features to limit the books to be catalogued. In addition, if you select multiple books using the mouse,
-       only those books will be added to the catalog. If you generate the catalog in an e-book format such as EPUB or MOBI,
-       the next time you connect your e-book reader, the catalog will be automatically sent to the device.
-       For details on how catalogs work, see `here <http://www.mobileread.com/forums/showthread.php?p=755468#post755468>`.
+    3. **Create a catalog of the books in your calibre library**: Allows you to generate a complete listing of the books in your library, including all metadata,
+       in several formats such as XML, CSV, BiBTeX, EPUB and MOBI. The catalog will contain all the books currently showing in the library view.
+       This allows you to use the search features to limit the books to be catalogued. In addition, if you select multiple books using the mouse,
+       only those books will be added to the catalog. If you generate the catalog in an ebook format such as EPUB or MOBI,
+       the next time you connect your ebook reader the catalog will be automatically sent to the device.
+       For more information on how catalogs work, read the `catalog creation tutorial <http://www.mobileread.com/forums/showthread.php?p=755468#post755468>`_
+       at MobileRead.
 
 .. _view:
 
@@ -112,7 +114,7 @@ View
 .. |vi| image:: images/view.png
     :class: float-right-img
 
-|vi| The :guilabel:`View` action displays the book in an ebook viewer program. |app| has a builtin viewer for the most e-book formats.
+|vi| The :guilabel:`View` action displays the book in an ebook viewer program. |app| has a built-in viewer for many ebook formats.
 For other formats it uses the default operating system application. You can configure which formats should open with the internal viewer via
 Preferences->Behavior. If a book has more than one format, you can view a particular format by clicking the down arrow
 on the right of the :guilabel:`View` button.
@@ -130,11 +132,11 @@ Send to device
     1. **Send to main memory**: The selected books are transferred to the main memory of the ebook reader.
     2. **Send to card (A)**: The selected books are transferred to the storage card (A) on the ebook reader.
     3. **Send to card (B)**: The selected books are transferred to the storage card (B) on the ebook reader.
-    4. **Send and delete from library>**: The selected books are transferred to the selected storage location on the device, and then **deleted** from the Library.
-    5. **Send Specific format>**: The selected books are transferred to the selected storage location on the device, in the format that you specify.
-    6. **Eject device**: The device is detached from |app|.
-    7. **Set default send to device action>**: This action allows you to Specify which of the option 1) through 6) above will be the default action when you click the main button.
-    8. **Fetch Annotations**: This is an experimental action which will transfer annotations you may have made on an ebook on your device, and add those annotations to the comments metadata of the book in the |app| library
+    4. **Send specific format to**: The selected books are transferred to the selected storage location on the device, in the format that you specify.
+    5. **Eject device**: Detaches the device from |app|.
+    6. **Set default send to device action**: Allows you to specify which of the options, 1 through 5 above or 7 below, will be the default action when you click the main button.
+    7. **Send and delete from library**: The selected books are transferred to the selected storage location on the device and then **deleted** from the Library.
+    8. **Fetch Annotations (experimental)**: Transfers annotations you may have made on an ebook on your device to the comments metadata of the book in the |app| library.
 
 You can control the file name and folder structure of files sent to the device by setting up a template in
 :guilabel:`Preferences->Import/Export->Sending books to devices`. Also see :ref:`templatelangcalibre`.
@@ -152,9 +154,9 @@ The :guilabel:`Fetch news` action uses simple recipes (10-15 lines of code) for
 
 The :guilabel:`Fetch news` action has three variations, accessed by clicking the down arrow on the right of the button.
 
-    1. **Schedule news download**: This action allows you to schedule the download of of your selected news sources from a list of hundreds of available.  Scheduling can be set individually for each news source you select and the scheduling is flexible allowing you to select specific days of the week or a frequency of days between downloads.
-    2. **Add a custom news service**: This action allows you to create a simple recipe for downloading news from a custom news site that you wish to access.  Creating the recipe can be as simple as specifying an RSS news feed URL, or you can be more prescriptive by creating python based code for the task, see :ref:`news`.
-    3. **Download all scheduled news sources**:  This action causes |app| to immediately begin to download all news sources that you have previously scheduled.
+    1. **Schedule news download**: Allows you to schedule the download of of your selected news sources from a list of hundreds available.  Scheduling can be set individually for each news source you select and the scheduling is flexible allowing you to select specific days of the week or a frequency of days between downloads.
+    2. **Add a custom news source**: Allows you to create a simple recipe for downloading news from a custom news site that you wish to access.  Creating the recipe can be as simple as specifying an RSS news feed URL, or you can be more prescriptive by creating Python-based code for the task. For more information see :ref:`news`.
+    3. **Download all scheduled news sources**: Causes |app| to immediately begin downloading all news sources that you have scheduled.
 
 
 .. _library:
@@ -164,16 +166,16 @@ Library
 .. |lii| image:: images/library.png
     :class: float-right-img
 
-|lii| The :guilabel:`Library` action allows you to create, switch between, rename or remove a Library.  |app| allows you to create as many libraries as you wish. You could for instance create a fiction library, a non fiction library, a foreign language library, a project library, basically any structure that suits your needs. Libraries are the highest organizational structure within |app|, each library has its own set of books, tags, categories and base storage location.
+|lii| The :guilabel:`Library` action allows you to create, switch between, rename or remove a Library.  |app| allows you to create as many libraries as you wish. You could, for instance, create a fiction library, a non-fiction library, a foreign language library, a project library, or any structure that suits your needs. Libraries are the highest organizational structure within |app|. Each library has its own set of books, tags, categories and base storage location.
 
-    1. **Switch/Create library**: This action allows you to; a) connect to a pre-existing |app| library at another location from your currently open library, b) Create and empty library at a new location or, c) Move the current Library to a newly specified location.
-    2. **Quick Switch**: This action allows you to switch between libraries that have been registered or created within |app|.
-    3. **Rename Library**: This action allows you to rename a Library.
-    4. **Remove Library**: This action allows you to unregister a library from |app|.
-    5. **<library name>**: Actions 5, 6 etc .. give you immediate switch access between multiple Libraries that you have created or attached to. This list contains only the 5 most frequently used libraries. For the complete list, use the Quick Switch menu.
-    6. **Library Maintenance**: This action allows you to check the current library for data consistency issues and restore the current libraries' database from backups.
+    1. **Switch/create library...**: Allows you to; a) connect to a pre-existing |app| library at another location, b) create an empty library at a new location or, c) move the current library to a newly specified location.
+    2. **Quick switch**: Allows you to switch between libraries that have been registered or created within |app|.
+    3. **Rename library**: Allows you to rename a Library.
+    4. **Delete library**: Allows you to unregister a library from |app|.
+    5. **<library name>**: Actions 5, 6 etc... give you immediate switch access between multiple libraries that you have created or attached to. This list contains only the 5 most frequently used libraries. For the complete list, use the Quick Switch menu.
+    6. **Library maintenance**: Allows you to check the current library for data consistency issues and restore the current library's database from backups.
 
-.. note:: Metadata about your ebooks like title/author/tags/etc. is stored in a single file in your |app| library folder called metadata.db. If this file gets corrupted (a very rare event), you can lose the metadata. Fortunately, |app| automatically backs up the metadata for every individual book in the book's folder as an .opf file. By using the Restore Library action under Library Maintenance described above, you can have |app| rebuild the metadata.db file from the individual .opf files for you. 
+.. note:: Metadata about your ebooks, e.g. title, author, and tags, is stored in a single file in your |app| library folder called metadata.db. If this file gets corrupted (a very rare event), you can lose the metadata. Fortunately, |app| automatically backs up the metadata for every individual book in the book's folder as an OPF file. By using the Restore Library action under Library Maintenance described above, you can have |app| rebuild the metadata.db file from the individual OPF files for you. 
 
 You can copy or move books between different libraries (once you have more than one library setup) by right clicking on the book and selecting the action :guilabel:`Copy to library`.
 
@@ -185,7 +187,7 @@ Device
     :class: float-right-img
 
 |dvi| The :guilabel:`Device` action allows you to view the books in the main memory or storage cards of your device, or to eject the device (detach it from |app|).
-This icon shows up automatically on the main |app| toolbar when you connect a supported device. You can click on it to see the books on your device. You can also drag  and drop books from your |app| library onto the icon to transfer them to your device. Conversely, you can drag and drop books from your device onto the |app| icon on the toolbar to transfer books from your device to the |app| library.
+This icon shows up automatically on the main |app| toolbar when you connect a supported device. You can click on it to see the books on your device. You can also drag and drop books from your |app| library onto the icon to transfer them to your device. Conversely, you can drag and drop books from your device onto the library icon on the toolbar to transfer books from your device to the |app| library.
 
 
 .. _save_to_disk:
@@ -199,7 +201,7 @@ Save to disk
 
 .. _save_to_disk_multiple:
 
-    1. **Save to disk**: This will save the selected books to disk organized in directories. The directory structure looks like::
+    1. **Save to disk**: Saves the selected books to disk organized in directories. The directory structure looks like::
 
             Author_(sort)
                 Title
@@ -210,17 +212,15 @@ Save to disk
 
 .. _save_to_disk_single:
 
-    2. **Save to disk in a single directory**: The selected books are saved to disk in a single directory.
+    2. **Save to disk in a single directory**: Saves the selected books to disk in a single directory.
 
-    For 1. and 2. All available formats as well as metadata is stored to disk for each selected book. Metadata is stored in an OPF file.
+    For 1. and 2., all available formats, as well as metadata, are stored to disk for each selected book. Metadata is stored in an OPF file. Saved books can be re-imported to the library without any loss of information by using the :ref:`Add books <add_books>` action.
 
-    Saved books can be re-imported to the library without any loss of information by using the :ref:`Add books <add_books>` action.
+    3. **Save only *<your preferred>* format to disk**: Saves the selected books to disk in the directory structure as shown in (1.) but only in your preferred ebook format. You can set your preferred format in :guilabel:`Preferences->Behaviour->Preferred output format`
 
-    3. **Save only *<your preferred>* format to disk**: The selected books are saved to disk in the directory structure as shown in (1.) but only in your preferred ebook format you can set <your preferred> format in :guilabel:`Preferences->Behaviour->Preferred output format`
+    4. **Save only *<your preferred>* format to disk in a single directory**: Saves the selected books to disk in a single directory but only in your preferred ebook format. You can set your preferred format in :guilabel:`Preferences->Behaviour->Preferred output format`
 
-    4. **Save only *<your preferred>* format to disk in a single directory**: The selected books are saved to disk in a single directory but only in <your preferred> ebook format you can set <your preferred> format in :guilabel:`Preferences->Behaviour->Preferred output format`
-
-    5. **Save single format to disk ..**: The selected books are saved to disk in the directory structure as shown in (1.) but only in the format you select from the pop-out list.  There are currently 35 formats available and new ones are being added all the time.
+    5. **Save single format to disk...**: Saves the selected books to disk in the directory structure as shown in (1.) but only in the format you select from the pop-out list.
 
 .. _connect_share:
 
@@ -229,17 +229,17 @@ Connect/Share
 .. |csi| image:: images/connect_share.png
     :class: float-right-img
 
-|csi| The :guilabel:`Connect/Share` action allows you to manually connect to a device or folder on your computer, it also allows you to set up you |app| library for access via a web browser, or email.
+|csi| The :guilabel:`Connect/Share` action allows you to manually connect to a device or folder on your computer. It also allows you to set up you |app| library for access via a web browser or email.
 
     The :guilabel:`Connect/Share` action has four variations, accessed by clicking the down arrow on the right of the button.
 
-    1. **Connect to folder**: This action allows you to connect to any folder on your computer as though it were a device and use all the facilities |app| has for devices with that folder. Useful if your device cannot be supported by |app| but is available as a USB disk.
+    1. **Connect to folder**: Allows you to connect to any folder on your computer as though it were a device and use all the facilities |app| has for devices with that folder. Useful if your device cannot be supported by |app| but is available as a USB disk.
 
-    2. **Connect to iTunes**: Allows you to connect to your iTunes books database as though it were a device. Once the books are sent to iTunes, you can then use iTunes to make them available on your various iDevices. Useful if you would rather not have |app| send books to your iDevice directly.
+    2. **Connect to iTunes**: Allows you to connect to your iTunes books database as though it were a device. Once the books are sent to iTunes, you can use iTunes to make them available to your various iDevices. This is useful if you would rather not have |app| send books to your iDevice directly.
 
-    3. **Start Content Server**: This action causes |app| to start up its built-in web server.  When this is started, your |app| library will be accessible via a web browser from the internet (if you choose). You can configure how the web server is accessed by setting preferences at :guilabel:`Preferences->Sharing->Sharing over the net`
+    3. **Start Content Server**: Starts |app|'s built-in web server.  When started, your |app| library will be accessible via a web browser from the Internet (if you choose). You can configure how the web server is accessed by setting preferences at :guilabel:`Preferences->Sharing->Sharing over the net`
 
-    4. **Setup email based sharing of books**: This action allows you to setup |app| to share books (and news feeds) by email.  After setting up email addresses for this option |app| will send news updates and book updates to the entered email addresses. You can configure how the |app| sends email by setting preferences at :guilabel:`Preferences->Sharing->Sharing books by email`. Once you have setup one or more email addresses, this menu entry get replaced by menu entries to send books to the setup email addresses.
+    4. **Setup email based sharing of books**: Allows sharing of books and news feeds by email.  After setting up email addresses for this option, |app| will send news updates and book updates to the entered email addresses. You can configure how |app| sends email by setting preferences at :guilabel:`Preferences->Sharing->Sharing books by email`. Once you have set up one or more email addresses, this menu entry will be replaced by menu entries to send books to the configured email addresses.
 
 .. _remove_books:
 
@@ -248,29 +248,36 @@ Remove books
 .. |rbi| image:: images/remove_books.png
     :class: float-right-img
 
-|rbi| The :guilabel:`Remove books` action **deletes books permanently**, so use it with care. It is *context sensitive*, i.e. it depends on which :ref:`catalog <catalogs>` you have selected. If you have selected the :guilabel:`Library`, books will be removed from the library. If you have selected the ebook reader device, the books will be removed from the device. To remove only a particular format for a given book use the :ref:`edit_meta_information` action.  Remove books also has five variations which can be accessed by clicking the down arrow on the right side of the button.
+|rbi| The :guilabel:`Remove books` action **deletes books permanently**, so use it with care. It is *context sensitive*, i.e. it depends on which :ref:`catalog <catalogs>` you have selected. If you have selected the :guilabel:`Library`, books will be removed from the library. If you have selected the ebook reader device, books will be removed from the device. To remove only a particular format for a given book use the :ref:`edit_meta_information` action.  Remove books also has five variations which can be accessed by clicking the down arrow on the right side of the button.
 
-    1. **Remove Selected Books**: Allows you to **permanently** remove all books that are selected in the book list.
+    1. **Remove selected books**: Allows you to **permanently** remove all books that are selected in the book list.
 
-    2. **Remove files of a specified format from selected books..**: Allows you to **permanently** remove ebook files of a specified format, from books that are selected in the book list.
+    2. **Remove files of a specific format from selected books...**: Allows you to **permanently** remove ebook files of a specified format from books that are selected in the book list.
 
-    3. **Remove all files of a specified format, except..**: Allows you to **permanently** remove ebook files of a multiple formats except a given format, from books that are selected in the book list.
+    3. **Remove all formats from selected books, except...**: Allows you to **permanently** remove ebook files of any format except a specified format from books that are selected in the book list.
 
-    4. **Remove covers from selected books**: Allows you to **permanently** remove cover images files, from books that are selected in the book list.
+    3. **Remove all formats from selected books**: Allows you to **permanently** remove all ebook files from books that are selected in the book list. Only the metadata will remain.
 
-    5. **Remove matching books from device**: Allows you to remove ebook files from a connected device, that match the books that are selected in the book list.
+    4. **Remove covers from selected books**: Allows you to **permanently** remove cover image files from books that are selected in the book list.
+
+    5. **Remove matching books from device**: Allows you to remove ebook files from a connected device that match the books that are selected in the book list.
 
 .. note::
-    Note that when you use Remove books to delete books from your |app| library, the book record is permanently deleted, but, on (Windows and OS X) the files are placed into the recycle bin, so you can recover them if you change your mind.
+    Note that when you use Remove books to delete books from your |app| library, the book record is permanently deleted, but on Windows and OS X the files are placed into the recycle bin. This allows you to recover them if you change your mind.
 
 .. _configuration:
 
 Preferences
 ---------------
 .. |cbi| image:: images/preferences.png
+    :class: float-right-img
 
-The Preferences Action allows you to change the way various aspects of |app| work. To access it, click the |cbi|.
-You can also re-run the Welcome Wizard by clicking the arrow next to the preferences button.
+|cbi| The :guilabel:`Preferences` action allows you to change the way various aspects of |app| work. It has four variations, accessed by clicking the down arrow on the right of the button.
+
+    1. **Preferences**: Allows you to change the way various aspects of |app| work. Clicking the button also performs this action.
+    2. **Run welcome wizard**: Allows you to start the Welcome Wizard which appeared the first time you started |app|.
+    3. **Get plugins to enhance |app|**: Opens a new windows that shows plugins for |app|. These plugins are developed by third parties to extend |app|'s functionality.
+    4. **Restart in debug mode**: Allows you to enable a debugging mode that can assist the |app| developers in solving problems you encounter with the program. For most users this should remain disabled unless instructed by a developer to enable it.
 
 .. _catalogs:
 
@@ -281,12 +288,11 @@ Catalogs
 
 A *catalog* is a collection of books. |app| can manage two types of different catalogs:
 
-    1. **Library**: This is a collection of books stored in your |app| library on your computer
+    1. **Library**: This is a collection of books stored in your |app| library on your computer.
 
-    2. **Device**:  This is a collection of books stored in the main memory of your ebook reader. It will be available when you connect the reader to your computer.
-       - In addition, you can see the books on the storage card (if any) in your reader device.
+    2. **Device**:  This is a collection of books stored in your ebook reader. It will be available when you connect the reader to your computer.
 
-Many operations, like Adding books, deleting, viewing, etc. are context sensitive. So, for example, if you click the View button when you have the **Device** catalog selected, |app| will open the files on the device to view. If you have the **Library** catalog selected, files in your |app| library will be opened instead.
+Many operations, such as adding books, deleting, viewing, etc., are context sensitive. So, for example, if you click the View button when you have the **Device** catalog selected, |app| will open the files on the device to view. If you have the **Library** catalog selected, files in your |app| library will be opened instead.
 
 .. _search_sort:
 
@@ -297,7 +303,7 @@ Search & Sort
 
 The Search & Sort section allows you to perform several powerful actions on your book collections.
 
-    * You can sort them by title, author, date, rating etc. by clicking on the column titles. You can also sub-sort (i.e. sort on multiple columns). For example, if you click on the title column and then the author column, the book will be sorted by author and then all the entries for the same author will be sorted by title.
+    * You can sort them by title, author, date, rating, etc. by clicking on the column titles. You can also sub-sort, i.e. sort on multiple columns. For example, if you click on the title column and then the author column, the book will be sorted by author and then all the entries for the same author will be sorted by title.
 
     * You can search for a particular book or set of books using the search bar. More on that below.
 
@@ -327,21 +333,21 @@ are available in the LRF format. Some more examples::
     format:epub publisher:feedbooks.com
 
 Searches are by default 'contains'. An item matches if the search string appears anywhere in the indicated metadata.
-Two other kinds of searches are available: equality search and search using regular expressions.
+Two other kinds of searches are available: equality search and search using `regular expressions <http://en.wikipedia.org/wiki/Regular_expression>`_.
 
 Equality searches are indicated by prefixing the search string with an equals sign (=). For example, the query
 ``tag:"=science"`` will match "science", but not "science fiction" or "hard science". Regular expression searches are
-indicated by prefixing the search string with a tilde (~). Any python-compatible regular expression can
-be used. Note that backslashes use to escape special characters in reqular expressions must be doubled, because single backslashes will be removed during query parsing. For example, to match a literal parenthesis, you must enter ``\\(``. Regular expression searches are contains searches unless the expression contains anchors. 
+indicated by prefixing the search string with a tilde (~). Any `python-compatible regular expression <http://docs.python.org/library/re.html>`_ can
+be used. Note that backslashes used to escape special characters in reqular expressions must be doubled because single backslashes will be removed during query parsing. For example, to match a literal parenthesis you must enter ``\\(``. Regular expression searches are 'contains' searches unless the expression contains anchors. 
 
 Should you need to search for a string with a leading equals or tilde, prefix the string with a backslash. 
 
 Enclose search strings with quotes (") if the string contains parenthesis or spaces. For example, to search
-for the tag ``Science Fiction``, you would need to search for ``tag:"=science fiction"``. If you search for
-``tag:=science fiction``, you will find all books with the tag 'science' and containing the word 'fiction' in any
+for the tag ``Science Fiction`` you would need to search for ``tag:"=science fiction"``. If you search for
+``tag:=science fiction`` you will find all books with the tag 'science' and containing the word 'fiction' in any
 metadata.
 
-You can build advanced search queries easily using the :guilabel:`Advanced Search Dialog`, accessed by
+You can build advanced search queries easily using the :guilabel:`Advanced Search Dialog` accessed by
 clicking the button |sbi|.
 
 Available fields for searching are: ``tag, title, author, publisher, series, series_index, rating, cover,
@@ -354,10 +360,13 @@ The syntax for searching for dates is::
     pubdate:=2009 Will find all books published in 2009
 
 If the date is ambiguous, the current locale is used for date comparison. For example, in an mm/dd/yyyy
-locale, 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale, it is interpreted as 2 Jan 2009.  Some
+locale 2/1/2009 is interpreted as 1 Feb 2009. In a dd/mm/yyyy locale it is interpreted as 2 Jan 2009.  Some
 special date strings are available. The string ``today`` translates to today's date, whatever it is. The
 strings ``yesterday`` and ``thismonth`` also work. In addition, the string ``daysago`` can be used to compare
-to a date some number of days ago, for example: date:>10daysago, date:<=45daysago.
+to a date some number of days ago. For example::
+
+    date:>10daysago
+    date:<=45daysago
 
 You can search for books that have a format of a certain size like this::
 
@@ -369,7 +378,7 @@ Dates and numeric fields support the relational operators ``=`` (equals), ``>``
 Rating fields are considered to be numeric. For example, the search ``rating:>=3`` will find all books rated 3
 or higher.
 
-You can search for the number of items in multiple-valued fields such as tags). These searches begin with the character ``#``, then use the same syntax as numeric fields. For example, to find all books with more than 4 tags, use ``tags:#>4``. To find all books with exactly 10 tags, use ``tags:#=10``.
+You can search for the number of items in multiple-valued fields such as tags. These searches begin with the character ``#``, then use the same syntax as numeric fields. For example, to find all books with more than 4 tags use ``tags:#>4``. To find all books with exactly 10 tags use ``tags:#=10``.
 
 Series indices are searchable. For the standard series, the search name is 'series_index'. For
 custom series columns, use the column search name followed by _index. For example, to search the indices for a
@@ -378,7 +387,7 @@ Series indices are numbers, so you can use the relational operators described ab
 
 The special field ``search`` is used for saved searches. So if you save a search with the name
 "My spouse's books" you can enter ``search:"My spouse's books"`` in the search bar to reuse the saved
-search. More about saving searches, below.
+search. More about saving searches below.
 
 You can search for the absence or presence of a field using the special "true" and "false" values. For example::
 
@@ -418,9 +427,9 @@ Identifiers (e.g., isbn, doi, lccn etc) also use an extended syntax. First, note
 Saving searches
 -----------------
 
-|app| has a useful feature, it allows you to save a search you use frequently under a special name and then re-use that search with a single click. To do this, create your search, either by typing it in the search bar, or using the Tag Browser. Then, type the name you would like to give to the search in the Saved Searches box next to the search bar and click the plus icon next to the saved searches box to save the search.
+|app| allows you to save a frequently used search under a special name and then reuse that search with a single click. To do this, create your search either by typing it in the search bar or using the Tag Browser. Then type the name you would like to give to the search in the Saved Searches box next to the search bar. Click the plus icon next to the saved searches box to save the search.
 
-Now, you can access your saved search in the Tag Browser under "Searches". A single click will allow you to re-use any arbitrarily complex search easily, without needing to re-create it.
+Now you can access your saved search in the Tag Browser under "Searches". A single click will allow you to reuse any arbitrarily complex search easily, without needing to re-create it.
 
 .. _config_filename_metadata:
 
@@ -431,7 +440,7 @@ that you add to the library. The default regular expression is::
 
     title - author
 
-i.e., it will assumes that all character up to the first ``-`` are the title of the book and subsequent characters are the author of the book. For example, the filename::
+i.e., it assumes that all characters up to the first ``-`` are the title of the book and subsequent characters are the author of the book. For example, the filename::
 
     Foundation and Earth - Isaac Asimov.txt
 
@@ -447,7 +456,7 @@ Book Details
 .. image:: images/book_details.png
     :align: center
 
-The Book Details display shows you extra information and the cover for the currently selected book.
+The Book Details display shows extra information and the cover for the currently selected book.
 
 .. _jobs:
 
@@ -460,15 +469,19 @@ Tag Browser
 
 The Tag Browser allows you to easily browse your collection by Author/Tags/Series/etc. If you click on any item in the Tag Browser, for example the author name Isaac Asimov, then the list of books to the right is restricted to showing books by that author. You can click on category names as well. For example, clicking on "Series" will show you all books in any series.
 
-The first click on an item will restrict the list of books to those that contain/match the item. Continuing the above example, clicking on Isaac Asimov will show books by that author. Clicking again on the item will change what is shown, depending on whether the item has children (see sub-categories and hierarchical items below). Continuing the Isaac Asimov example, clicking again on Isaac Asimov will restrict the list of books to those not by Isaac Asimov. A third click will remove the restriction, showing all books. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could hold Ctrl and click on the tags History and Europe for find books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. Looking at what the Tag Browser generates is a good way to learn how to construct basic search expressions.
+The first click on an item will restrict the list of books to those that contain or match the item. Continuing the above example, clicking on Isaac Asimov will show books by that author. Clicking again on the item will change what is shown, depending on whether the item has children (see sub-categories and hierarchical items below). Continuing the Isaac Asimov example, clicking again on Isaac Asimov will restrict the list of books to those not by Isaac Asimov. A third click will remove the restriction, showing all books. If you hold down the Ctrl or Shift keys and click on multiple items, then restrictions based on multiple items are created. For example you could hold Ctrl and click on the tags History and Europe for finding books on European history. The Tag Browser works by constructing search expressions that are automatically entered into the Search bar. Looking at what the Tag Browser generates is a good way to learn how to construct basic search expressions.
 
 Items in the Tag browser have their icons partially colored. The amount of color depends on the average rating of the books in that category. So for example if the books by Isaac Asimov have an average of four stars, the icon for Isaac Asimov in the Tag Browser will be 4/5th colored. You can hover your mouse over the icon to see the average rating.
 
-The outer-level items in the tag browser such as Authors and Series are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the User Categories Editor (push the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the sub-category name; or by using the User Categories Editor by entering names like the Favorites example above.
+The outer-level items in the tag browser, such as Authors and Series, are called categories. You can create your own categories, called User Categories, which are useful for organizing items. For example, you can use the User Categories Editor (click the Manage User Categories button) to create a user category called Favorite Authors, then put the items for your favorites into the category. User categories can have sub-categories. For example, the user category Favorites.Authors is a sub-category of Favorites. You might also have Favorites.Series, in which case there will be two sub-categories under Favorites. Sub-categories can be created by right-clicking on a user category, choosing "Add sub-category to ...", and entering the sub-category name; or by using the User Categories Editor by entering names like the Favorites example above.
 
-You can search user categories in the same way as built-in categories, by clicking on them. There are four different searches cycled through by clicking: "everything matching an item in the category" indicated by a single green plus sign, "everything matching an item in the category or its sub-categories" indicated by two green plus signs, "everything not matching an item in the category" shown by a single red minus sign, and "everything not matching an item in the category or its sub-categories" shown by two red minus signs.
+You can search user categories in the same way as built-in categories, by clicking on them. There are four different searches cycled through by clicking:
+    1. "everything matching an item in the category" indicated by a single green plus sign.
+    2. "everything matching an item in the category or its sub-categories" indicated by two green plus signs.
+    3. "everything not matching an item in the category" shown by a single red minus sign.
+    4. "everything not matching an item in the category or its sub-categories" shown by two red minus signs.
 
-It is also possible to create hierarchies inside some of the text categories such as tags, series, and custom columns. These hierarchies show with the small triangle, permitting the sub-items to be hidden. To use hierarchies of items in a category, you must first go to Preferences / Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items. See :ref:`Managing subgroups of books, for example "genre" <subgroups-tutorial>` for more information.
+It is also possible to create hierarchies inside some of the text categories such as tags, series, and custom columns. These hierarchies show with the small triangle, permitting the sub-items to be hidden. To use hierarchies of items in a category, you must first go to Preferences->Look & Feel and enter the category name(s) into the "Categories with hierarchical items" box. Once this is done, items in that category that contain periods will be shown using the small triangle. For example, assume you create a custom column called "Genre" and indicate that it contains hierarchical items. Once done, items such as Mystery.Thriller and Mystery.English will display as Mystery with the small triangle next to it. Clicking on the triangle will show Thriller and English as sub-items. See :ref:`Managing subgroups of books, for example "genre" <subgroups-tutorial>` for more information.
 
 Hierarchical items (items with children) use the same four 'click-on' searches as user categories. Items that do not have children use two of the searches: "everything matching" and "everything not matching".
 
@@ -476,23 +489,23 @@ You can drag and drop items in the Tag browser onto user categories to add them
 
 There is a search bar at the top of the Tag Browser that allows you to easily find any item in the Tag Browser. In addition, you can right click on any item and choose one of several operations. Some examples are to hide the it, rename it, or open a "Manage x" dialog that allows you to manage items of that kind. For example, the "Manage Authors" dialog allows you to rename authors and control how their names are sorted.
 
-You can control how items are sorted in the Tag browser via the box at the bottom of the Tag Browser. You can choose to sort by name, average rating or popularity (popularity is the number of books with an item in your library; for example; the popularity of Isaac Asimov is the number of book sin your library by Isaac Asimov).
+You can control how items are sorted in the Tag browser via the box at the bottom of the Tag Browser. You can choose to sort by name, average rating or popularity (popularity is the number of books with an item in your library; for example, the popularity of Isaac Asimov is the number of books in your library by Isaac Asimov).
 
 Jobs
 -----
 .. image:: images/jobs.png
     :class: float-left-img
 
-The Jobs panel shows you the number of currently running jobs. Jobs are tasks that run in a separate process, they include converting ebooks and talking to your reader device. You can click on the jobs panel to access the list of jobs. Once a job has completed, by double-clicking it in the list, you can see a detailed log from that job. This is useful to debug jobs that may not have completed successfully.
+The Jobs panel shows the number of currently running jobs. Jobs are tasks that run in a separate process. They include converting ebooks and talking to your reader device. You can click on the jobs panel to access the list of jobs. Once a job has completed you can see a detailed log from that job by double-clicking it in the list. This is useful to debug jobs that may not have completed successfully.
 
 Keyboard Shortcuts
 ---------------------
 
-Calibre has several keyboard shortcuts to save you time and mouse movement. These shortcuts are active in the book list view (when you're not editing the details of a particular book), and most of them affect the title you have selected. The |app| e-book viewer has its own shortcuts, which can be customised by clicking the Preferences button in the viewer.
+Calibre has several keyboard shortcuts to save you time and mouse movement. These shortcuts are active in the book list view (when you're not editing the details of a particular book), and most of them affect the title you have selected. The |app| ebook viewer has its own shortcuts which can be customised by clicking the Preferences button in the viewer.
 
 .. note::
 
-    Note: The Calibre keyboard shortcuts do not require a modifier key (Command, Option, Control etc.), unless specifically noted. You only need to press the letter key, e.g. E to edit.
+    Note: The Calibre keyboard shortcuts do not require a modifier key (Command, Option, Control, etc.), unless specifically noted. You only need to press the letter key, e.g. E to edit.
 
 .. list-table:: Keyboard Shortcuts
     :widths: 10 100
@@ -535,7 +548,7 @@ Calibre has several keyboard shortcuts to save you time and mouse movement. Thes
     * - :kbd:`Alt+Shift+T`
       - Toggle Tag Browser
     * - :kbd:`Alt+A`
-      - Show books by the Same author as the current book
+      - Show books by the same author as the current book
     * - :kbd:`Alt+T`
       - Show books with the same tags as current book
     * - :kbd:`Alt+P`

src/calibre/manual/index.rst

@@ -3,23 +3,21 @@
 |app| User Manual
 ===============================================
 
-|app| is an e-book library manager. It can view, convert and catalog e-books in most of the major e-book formats. It can also talk to a few e-book reader devices. It can go out to the internet and fetch metadata for your books. It can download newspapers and convert them into e-books for convenient reading. It is cross platform, running on Linux, Windows and OS X. 
+|app| is an ebook library manager. It can view, convert and catalog ebooks in most of the major ebook formats. It can also talk to many ebook reader devices. It can go out to the Internet and fetch metadata for your books. It can download newspapers and convert them into ebooks for convenient reading. It is cross platform, running on Linux, Windows and OS X. 
 
-So you've just started |app|. What do you do now? Well, before |app| can do anything with your ebooks, it first has to know about them. So drag and drop a few e-book files into |app|, or click the "Add books" button and browse for the ebooks you want to work with. Once you've added the books, they will show up in the main view looking something like this:
+You've just started |app|. What do you do now? Before |app| can do anything with your ebooks, it first has to know about them. Drag and drop a few ebook files into |app|, or click the "Add books" button and browse for the ebooks you want to work with. Once you've added the books, they will show up in the main view looking something like this:
 
 .. image:: images/added_books.png
 
-Once you've admired the list of books you just added to your heart's content, you'll probably want to actually read one. In order to do that you'll have to convert the book to a format your reader understands. When first running |app|, the Welcome Wizard starts and it will have setup calibre for your reader device. Conversion is a breeze, just select the book you want to convert, and click the "Convert E-book" button. Ignore all the options for now and just click "OK". The little icon in the bottom right corner will start spinning. Once it's finished spinning, your converted book is ready. Click the "View" button to read the book.
+Once you've admired the list of books you just added to your heart's content, you'll probably want to read one. In order to do that you'll have to convert the book to a format your reader understands. When first running |app|, the Welcome Wizard starts and will set up calibre for your reader device. Conversion is a breeze. Just select the book you want to convert then click the "Convert books" button. Ignore all the options for now and click "OK". The little icon in the bottom right corner will start spinning. Once it's finished spinning, your converted book is ready. Click the "View" button to read the book.
 
-Now if you want to read the book on your reader, just connect it to the computer, wait till calibre detects it (10-20secs) and then click the "Send to device" button. Once the icon stops spinning again, disconnect your reader and read away! If you didn't convert the book in the previous step, |app| will auto convert it to the format your reader device understands. 
+If you want to read the book on your reader, connect it to the computer, wait till calibre detects it (10-20 seconds) and then click the "Send to device" button. Once the icon stops spinning again, disconnect your reader and read away! If you didn't convert the book in the previous step, |app| will auto convert it to the format your reader device understands. 
 
-To get started with more advanced usage, you should read about the :ref:`Graphical User Interface <gui>`. For even more power and versatility, learn the :ref:`Command Line Interface <cli>`. 
-
-You will find the list of :ref:`Frequently Asked Questions <faq>` useful as well. 
+To get started with more advanced usage, you should read about the :ref:`Graphical User Interface <gui>`. For even more power and versatility, learn the :ref:`Command Line Interface <cli>`. You will find the list of :ref:`Frequently Asked Questions <faq>` useful as well. 
 
 .. only:: online
     
-    An e-book version of this User Manual is available in `EPUB format <calibre.epub>`_. 
+    An ebook version of this user manual is available in `EPUB format <calibre.epub>`_. 
     
 Sections
 ------------
@@ -56,7 +54,7 @@ Adding your favorite news website to |app|
 
    news
 
-The |app| e-book viewer
+The |app| ebook viewer
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. toctree::
@@ -64,7 +62,7 @@ The |app| e-book viewer
 
    viewer
 
-Customizing |app|'s e-book conversion
+Customizing |app|'s ebook conversion
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. toctree::
@@ -72,7 +70,7 @@ Customizing |app|'s e-book conversion
 
    conversion
 
-Editing e-book metadata
+Editing ebook metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. toctree::

src/calibre/manual/metadata.rst

@@ -2,14 +2,14 @@
 
 .. _metadata:
 
-Editing E-book Metadata
+Editing Ebook Metadata
 ========================
 
 .. contents:: Contents
   :depth: 2
   :local:
 
-E-books come in all shapes and sizes and more often than not, their metadata (things like title/author/series/publisher) is incomplete or incorrect.
+Ebooks come in all shapes and sizes and more often than not, their metadata (things like title/author/series/publisher) is incomplete or incorrect.
 The simplest way to change metadata in |app| is to simply double click on an entry and type in the correct replacement.
 For more sophisticated, "power editing" use the edit metadata tools discussed below.
 

src/calibre/manual/news.rst

@@ -5,7 +5,7 @@
 Adding your favorite news website
 ==================================
 
-|app| has a powerful, flexible and easy-to-use framework for downloading news from the internet and converting it into an e-book. In the following, I will show you by means of examples, how to get news from various websites.
+|app| has a powerful, flexible and easy-to-use framework for downloading news from the Internet and converting it into an ebook. The following will show you, by means of examples, how to get news from various websites.
 
 To gain an understanding of how to use the framework, follow the examples in the order listed below:
 
@@ -23,27 +23,27 @@ If your news source is simple enough, |app| may well be able to fetch it complet
 portfolio.com
 ~~~~~~~~~~~~~~~~~~~
 
-*portfolio.com* is the website for *Condé Nast Portfolio*, a business related magazine. In order to download articles from the magazine and convert them to e-books, we rely on the :term:`RSS` feeds of portfolio.com. A list of such feeds is available at http://www.portfolio.com/rss/.
+*portfolio.com* is the website for *Condé Nast Portfolio*, a business related magazine. In order to download articles from the magazine and convert them to ebooks, we rely on the :term:`RSS` feeds of portfolio.com. A list of such feeds is available at http://www.portfolio.com/rss/.
 
 Lets pick a couple of feeds that look interesting:
 
     #. Business Travel: http://feeds.portfolio.com/portfolio/businesstravel
     #. Tech Observer: http://feeds.portfolio.com/portfolio/thetechobserver
 
-I got the URLs by clicking the little orange RSS icon next to each feed name. To make |app| download the feeds and convert them into an e-book, you should click the :guilabel:`Fetch news` button and then the :guilabel:`Add a custom news source` menu item. A dialog similar to that shown below should open up.
+I got the URLs by clicking the little orange RSS icon next to each feed name. To make |app| download the feeds and convert them into an ebook, you should click the :guilabel:`Fetch news` button and then the :guilabel:`Add a custom news source` menu item. A dialog similar to that shown below should open up.
 
 .. image:: images/custom_news.png
     :align: center
 
-First enter ``Portfolio`` into the :guilabel:`Recipe title` field. This will be the title of the e-book that will be created from the articles in the above feeds. 
+First enter ``Portfolio`` into the :guilabel:`Recipe title` field. This will be the title of the ebook that will be created from the articles in the above feeds. 
 
 The next two fields (:guilabel:`Oldest article` and :guilabel:`Max. number of articles`) allow you some control over how many articles should be downloaded from each feed, and they are pretty self explanatory.
 
 To add the feeds to the recipe, enter the feed title and the feed URL and click the :guilabel:`Add feed` button. Once you have added both feeds, simply click the :guilabel:`Add/update recipe` button and you're done! Close the dialog.
 
-To test your new :term:`recipe`, click the :guilabel:`Fetch news` button and in the :guilabel:`Custom news sources` sub-menu click :guilabel:`Portfolio`. After a couple of minutes, the newly downloaded Portfolio e-book will appear in the main library view (if you have your reader connected, it will be put onto the reader instead of into the library). Select it and hit the :guilabel:`View` button to read!
+To test your new :term:`recipe`, click the :guilabel:`Fetch news` button and in the :guilabel:`Custom news sources` sub-menu click :guilabel:`Portfolio`. After a couple of minutes, the newly downloaded Portfolio ebook will appear in the main library view (if you have your reader connected, it will be put onto the reader instead of into the library). Select it and hit the :guilabel:`View` button to read!
 
-The reason this worked so well, with so little effort is that *portfolio.com* provides *full-content* :term:`RSS` feeds, i.e., the article content is embedded in the feed itself. For most news sources that provide news in this fashion, with *full-content* feeds, you don't need any more effort to convert them to e-books. Now we will look at a news source that does not provide full content feeds. In such feeds, the full article is a webpage and the feed only contains a link to the webpage with a short summary of the article. 
+The reason this worked so well, with so little effort is that *portfolio.com* provides *full-content* :term:`RSS` feeds, i.e., the article content is embedded in the feed itself. For most news sources that provide news in this fashion, with *full-content* feeds, you don't need any more effort to convert them to ebooks. Now we will look at a news source that does not provide full content feeds. In such feeds, the full article is a webpage and the feed only contains a link to the webpage with a short summary of the article. 
 
 .. _bbc:
 
@@ -55,19 +55,19 @@ Lets try the following two feeds from *The BBC*:
     #. News Front Page: http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml
     #. Science/Nature: http://newsrss.bbc.co.uk/rss/newsonline_world_edition/science/nature/rss.xml
 
-Follow the procedure outlined in :ref:`portfolio` to create a recipe for *The BBC* (using the feeds above). Looking at the downloaded e-book, we see that |app| has done a creditable job of extracting only the content you care about from each article's webpage. However, the extraction process is not perfect. Sometimes it leaves in undesirable content like menus and navigation aids or it removes content that should have been left alone, like article headings. In order, to have perfect content extraction, we will need to customize the fetch process, as described in the next section. 
+Follow the procedure outlined in :ref:`portfolio` to create a recipe for *The BBC* (using the feeds above). Looking at the downloaded ebook, we see that |app| has done a creditable job of extracting only the content you care about from each article's webpage. However, the extraction process is not perfect. Sometimes it leaves in undesirable content like menus and navigation aids or it removes content that should have been left alone, like article headings. In order, to have perfect content extraction, we will need to customize the fetch process, as described in the next section. 
 
 Customizing the fetch process
 --------------------------------
 
 When you want to perfect the download process, or download content from a particularly complex website, you can avail yourself of all the power and flexibility of the :term:`recipe` framework. In order to do that, in the :guilabel:`Add custom news sources` dialog, simply click the :guilabel:`Switch to Advanced mode` button.
 
-The easiest and often most productive customization is to use the print version of the online articles. The print version typically has much less cruft and translates much more smoothly to an e-book. Let's try to use the print version of the articles from *The BBC*.
+The easiest and often most productive customization is to use the print version of the online articles. The print version typically has much less cruft and translates much more smoothly to an ebook. Let's try to use the print version of the articles from *The BBC*.
 
 Using the print version of bbc.co.uk
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The first step is to look at the e-book we downloaded previously from :ref:`bbc`. At the end of each article, in the e-book is a little blurb telling you where the article was downloaded from. Copy and paste that URL into a browser. Now on the article webpage look for a link that points to the "Printable version". Click it to see the print version of the article. It looks much neater! Now compare the two URLs. For me they were:
+The first step is to look at the ebook we downloaded previously from :ref:`bbc`. At the end of each article, in the ebook is a little blurb telling you where the article was downloaded from. Copy and paste that URL into a browser. Now on the article webpage look for a link that points to the "Printable version". Click it to see the print version of the article. It looks much neater! Now compare the two URLs. For me they were:
 
     Article URL
         http://news.bbc.co.uk/2/hi/science/nature/7312016.stm
@@ -98,7 +98,7 @@ This is python, so indentation is important. After you've added the lines, it sh
 
 In the above, ``def print_version(self, url)`` defines a *method* that is called by |app| for every article. ``url`` is the URL of the original article. What ``print_version`` does is take that url and replace it with the new URL that points to the print version of the article. To learn about `python <http://www.python.org>`_ see the `tutorial <http://docs.python.org/tut/>`_.
 
-Now, click the :guilabel:`Add/update recipe` button and your changes will be saved. Re-download the e-book. You should have a much improved e-book. One of the problems with the new version is that the fonts on the print version webpage are too small. This is automatically fixed when converting to an e-book, but even after the fixing process, the font size of the menus and navigation bar to become too large relative to the article text. To fix this, we will do some more customization, in the next section.
+Now, click the :guilabel:`Add/update recipe` button and your changes will be saved. Re-download the ebook. You should have a much improved ebook. One of the problems with the new version is that the fonts on the print version webpage are too small. This is automatically fixed when converting to an ebook, but even after the fixing process, the font size of the menus and navigation bar to become too large relative to the article text. To fix this, we will do some more customization, in the next section.
 
 Replacing article styles
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -131,7 +131,7 @@ This can be removed by adding::
 
     remove_tags    = [dict(name='div', attrs={'class':'footer'})]
 
-to the recipe. Finally, lets replace some of the :term:`CSS` that we disabled earlier, with our own :term:`CSS` that is suitable for conversion to an e-book::
+to the recipe. Finally, lets replace some of the :term:`CSS` that we disabled earlier, with our own :term:`CSS` that is suitable for conversion to an ebook::
 
     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }' 
 
@@ -234,7 +234,7 @@ We see several new features in this :term:`recipe`. First, we have::
 
     timefmt = ' [%a, %d %b, %Y]'
 
-This sets the displayed time on the front page of the created e-book to be in the format,
+This sets the displayed time on the front page of the created ebook to be in the format,
 ``Day, Day_Number Month, Year``. See :attr:`timefmt <calibre.web.feeds.news.BasicNewsRecipe.timefmt>`.
 
 Then we see a group of directives to cleanup the downloaded :term:`HTML`::
@@ -254,7 +254,7 @@ The next interesting feature is::
 
 ``needs_subscription = True`` tells |app| that this recipe needs a username and password in order to access the content. This causes, |app| to ask for a username and password whenever you try to use this recipe. The code in :meth:`calibre.web.feeds.news.BasicNewsRecipe.get_browser` actually does the login into the NYT website. Once logged in, |app| will use the same, logged in, browser instance to fetch all content. See `mechanize <http://wwwsearch.sourceforge.net/mechanize/>`_ to understand the code in ``get_browser``.
 
-The next new feature is the :meth:`calibre.web.feeds.news.BasicNewsRecipe.parse_index` method. Its job is to go to http://www.nytimes.com/pages/todayspaper/index.html and fetch the list of articles that appear in *todays* paper. While more complex than simply using :term:`RSS`, the recipe creates an e-book that corresponds very closely to the days paper. ``parse_index`` makes heavy use of `BeautifulSoup <http://www.crummy.com/software/BeautifulSoup/documentation.html>`_ to parse the daily paper webpage.
+The next new feature is the :meth:`calibre.web.feeds.news.BasicNewsRecipe.parse_index` method. Its job is to go to http://www.nytimes.com/pages/todayspaper/index.html and fetch the list of articles that appear in *todays* paper. While more complex than simply using :term:`RSS`, the recipe creates an ebook that corresponds very closely to the days paper. ``parse_index`` makes heavy use of `BeautifulSoup <http://www.crummy.com/software/BeautifulSoup/documentation.html>`_ to parse the daily paper webpage.
 
 The final new feature is the :meth:`calibre.web.feeds.news.BasicNewsRecipe.preprocess_html` method. It can be used to perform arbitrary transformations on every downloaded HTML page. Here it is used to bypass the ads that the nytimes shows you before each article.
 

src/calibre/manual/qthelp.py

@@ -51,19 +51,19 @@ collection_template = '''\
 '''
 
 ABOUT='''\
-calibre is the one stop solution for all your e-book needs. It was created
-originally by Kovid Goyal, to help him manage his e-book collection and is now
-very actively developed by an international community of e-book enthusiasts.
+calibre is the one stop solution for all your ebook needs. It was created
+originally by Kovid Goyal, to help him manage his ebook collection and is now
+very actively developed by an international community of ebook enthusiasts.
 
-Its goal is to empower you, the user, to do whatever you like with the e-books
+Its goal is to empower you, the user, to do whatever you like with the ebooks
 in your collection. You can convert them to many different formats, read them
 on your computer, send them to many different devices, edit their metadata
 and covers, etc.
 
 calibre also allows you to download news from a variety of different sources all
-over the internet and read conveniently in e-books form. In keeping with its
+over the Internet and read conveniently in ebooks form. In keeping with its
 philosophy of empowering the user, it has a simple system to allow you to add
-your own favorite news sources. In fact, most the builtin news sources in
+your own favorite news sources. In fact, most the built-in news sources in
 calibre were originally contributed by users.
 '''
 

src/calibre/manual/template_lang.rst

@@ -179,7 +179,7 @@ The example shows several things:
 
     * program mode is used if the expression begins with ``:'`` and ends with ``'``. Anything else is assumed to be single-function.
     * the variable ``$`` stands for the field the expression is operating upon, ``#series`` in this case.
-    * functions must be given all their arguments. There is no default value. For example, the standard builtin functions must be given an additional initial parameter indicating the source field, which is a significant difference from single-function mode.
+    * functions must be given all their arguments. There is no default value. For example, the standard built-in functions must be given an additional initial parameter indicating the source field, which is a significant difference from single-function mode.
     * white space is ignored and can be used anywhere within the expression.
     * constant strings are enclosed in matching quotes, either ``'`` or ``"``.
     

src/calibre/manual/template_ref_generate.py

@@ -11,10 +11,10 @@ PREAMBLE = '''\
 
 .. _templaterefcalibre:
 
-Reference for all builtin template language functions
+Reference for all built-in template language functions
 ========================================================
 
-Here, we document all the builtin functions available in the |app| template language. Every function is implemented as a class in python and you can click the source links to see the source code, in case the documentation is insufficient. The functions are arranged in logical groups by type.
+Here, we document all the built-in functions available in the |app| template language. Every function is implemented as a class in python and you can click the source links to see the source code, in case the documentation is insufficient. The functions are arranged in logical groups by type.
 
 .. contents::
     :depth: 2

src/calibre/manual/tutorials.rst

@@ -6,7 +6,7 @@
 Tutorials
 =======================================================
 
-Here you will find tutorials to get you started using |app|'s more advanced features, like XPath and templates.
+Here you will find tutorials to get you started using |app|'s more advanced features, such as XPath and templates.
 
 .. toctree::
    :maxdepth: 1

src/calibre/manual/viewer.rst

@@ -2,10 +2,10 @@
 
 .. _viewer:
 
-The E-book Viewer
+The Ebook Viewer
 =============================
 
-|app| includes a built-in E-book viewer that can view all the major e-book formats. 
+|app| includes a built-in ebook viewer that can view all the major ebook formats. 
 The viewer is highly customizable and has many advanced features. 
 
 .. contents::
@@ -16,11 +16,11 @@ Starting the viewer
 --------------------
 
 You can view any of the books in your |app| library by selecting the book and pressing the View button. This
-will open up the book in the e-book viewer. You can also launch the viewer by itself, from the Start menu in windows
+will open up the book in the ebook viewer. You can also launch the viewer by itself from the Start menu in Windows
 or using the command :command:`ebook-viewer` in Linux and OS X (you have to install the command line tools on OS X
 first by going to :guilabel:`Preferences->Advanced->Miscellaneous`).
 
-Navigating around an e-book
+Navigating around an ebook
 -----------------------------
 
 .. |pni| image:: images/prev_next.png
@@ -35,7 +35,7 @@ Navigating around an e-book
 
 
 You can "turn pages" in a book by using the :guilabel:`Page Next` and :guilabel:`Page Previous` buttons |pni|, or by pressing
-the Page Down/Page Up keys. Unlike most e-book viewers, |app| does not force you to view books in paged mode. You can
+the Page Down/Page Up keys. Unlike most ebook viewers, |app| does not force you to view books in paged mode. You can
 scroll by amounts less than a page by using the scroll bar or various customizable keyboard shortcuts. 
 
 Bookmarks
@@ -43,27 +43,27 @@ Bookmarks
 
 When you are in the middle of a book and close the viewer, it will remember where you stopped reading and return there
 the next time you open the book. You can also set bookmarks in the book by using the Bookmark button |bookmi|. When viewing EPUB format
-books, these bookmarks are actually saved in the EPUB file itself, so you can add bookmarks, then send the file to a friend and
-when they open the file, they will be able to see your bookmarks.
+books, these bookmarks are actually saved in the EPUB file itself. You can add bookmarks, then send the file to a friend.
+When they open the file, they will be able to see your bookmarks.
 
 Table of Contents
 ^^^^^^^^^^^^^^^^^^^^
 
 If the book you are reading defines a Table of Contents, you can access it by pressing the Table of Contents button |toci|.
-This will bring up a list of sections in the book and you can click on any of them to jump to that portion of the book.
+This will bring up a list of sections in the book. You can click on any of them to jump to that portion of the book.
 
 Navigating by location
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-E-books, unlike paper books have no concept of pages. Instead,
+Ebooks, unlike paper books, have no concept of pages. Instead,
 as you read through the book, you will notice that your position in the book is displayed in the upper left corner in a box
 like this |navposi|. This is both your current position and the total length of the book. These numbers are independent of the screen size and font
-size you are viewing the boko at, and they play a similar role to page numbers in paper books.
+size you are viewing the book at, and they play a similar role to page numbers in paper books.
 You can enter any number you like to go to the corresponding location in the book. 
 
 |app| also has a very handy
 reference mode. You can turn it on by clicking the Reference Mode button |refmi|. Once you do this, every time you move your
-mouse over a paragraph, calibre will display a unique number made up of the section and paragraph numbers. 
+mouse over a paragraph, |app| will display a unique number made up of the section and paragraph numbers. 
 
 .. image:: images/ref_mode.png
     :align: center
@@ -72,8 +72,8 @@ You can use this number to unambiguously refer to parts of the books when discus
 in other works. You can enter these numbers in the box marked Go to at the top of the window to go to a particular
 reference location. 
 
-If you click on links inside the e-book to take you to different parts of the book, like an endnote, you can use the back and forward buttons 
-in the top left corner to return to where you were. These button behave just like those in a web browser. 
+If you click on links inside the ebook to take you to different parts of the book, such as an endnote, you can use the back and forward buttons 
+in the top left corner to return to where you were. These buttons behave just like those in a web browser. 
 
 Customizing the look and feel of your reading experience
 ------------------------------------------------------------
@@ -90,7 +90,7 @@ by the viewer to ones you like as well as the default font size when the viewer
 
 More advanced customization can be achieved by the User Stylesheet setting. This is a stylesheet you can set that will be applied
 to every book. Using it you can do things like have white text on a black background, change paragraph styles, text justification, etc.
-For examples if custom stylesheets used by |app|'s users, see `the forums <http://www.mobileread.com/forums/showthread.php?t=51500>`_.
+For examples of custom stylesheets used by |app|'s users, see `the forums <http://www.mobileread.com/forums/showthread.php?t=51500>`_.
 
 Dictionary lookup
 -------------------
@@ -101,6 +101,6 @@ server at ``dict.org`` to lookup words. The definition is displayed in a small b
 Copying text and images
 -------------------------
 
-You can select text and images by dragging the content with your mouse and then right click to copy to the clipboard.
+You can select text and images by dragging the content with your mouse and then right clicking to copy to the clipboard.
 The copied material can be pasted into another application as plain text and images.
 

src/calibre/ptempfile.py

@@ -114,7 +114,17 @@ def PersistentTemporaryDirectory(suffix='', prefix='', dir=None):
     '''
     if dir is None:
         dir = base_dir()
+    try:
         tdir = tempfile.mkdtemp(suffix, __appname__+"_"+ __version__+"_" +prefix, dir)
+    except ValueError:
+        global _base_dir
+        from calibre.constants import filesystem_encoding
+        base_dir()
+        if not isinstance(_base_dir, unicode):
+            _base_dir = _base_dir.decode(filesystem_encoding)
+        dir = dir.decode(filesystem_encoding)
+        tdir = tempfile.mkdtemp(suffix, __appname__+"_"+ __version__+"_" +prefix, dir)
+
     atexit.register(remove_dir, tdir)
     return tdir