Merge from trunk

recipes/le_monde.recipe

@@ -1,8 +1,9 @@
 __license__   = 'GPL v3'
-__copyright__ = '2011'
+__copyright__ = '2012'
 '''
 lemonde.fr
 '''
+import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
 class LeMonde(BasicNewsRecipe):
@@ -24,7 +25,7 @@ class LeMonde(BasicNewsRecipe):
                     .ariane{font-size:xx-small;}
                     .source{font-size:xx-small;}
                     #.href{font-size:xx-small;}
-                    .LM_caption{color:#666666; font-size:x-small;}
+                    #.figcaption style{color:#666666; font-size:x-small;}
                     #.main-article-info{font-family:Arial,Helvetica,sans-serif;}
                     #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
                     #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
@@ -40,8 +41,88 @@ class LeMonde(BasicNewsRecipe):
 
     remove_empty_feeds = True
 
-    auto_cleanup = True
+    filterDuplicates = True
 
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    preprocess_regexps = [
+        (re.compile(r'([0-9])%'), lambda m: m.group(1) + ' %'),
+        (re.compile(r'([0-9])([0-9])([0-9]) ([0-9])([0-9])([0-9])'), lambda m: m.group(1) + m.group(2) + m.group(3) + ' ' + m.group(4) + m.group(5) + m.group(6)),
+        (re.compile(r'([0-9]) ([0-9])([0-9])([0-9])'), lambda m: m.group(1) + ' ' + m.group(2) + m.group(3) + m.group(4)),
+        (re.compile(r'<span>'), lambda match: ' <span>'),
+        (re.compile(r'\("'), lambda match: '(&laquo;&nbsp;'),
+        (re.compile(r'"\)'), lambda match: '&nbsp;&raquo;)'),
+        (re.compile(r'&ldquo;'), lambda match: '(&laquo;&nbsp;'),
+        (re.compile(r'&rdquo;'), lambda match: '&nbsp;&raquo;)'),
+        (re.compile(r'>\''), lambda match: '>&lsquo;'),
+        (re.compile(r' \''), lambda match: ' &lsquo;'),
+        (re.compile(r' &quot;'), lambda match: ' &laquo;&nbsp;'),
+        (re.compile(r'>&quot;'), lambda match: '>&laquo;&nbsp;'),
+        (re.compile(r'&quot;<'), lambda match: '&nbsp;&raquo;<'),
+        (re.compile(r'&quot; '), lambda match: '&nbsp;&raquo; '),
+        (re.compile(r'&quot;,'), lambda match: '&nbsp;&raquo;,'),
+        (re.compile(r'\''), lambda match: '&rsquo;'),
+        (re.compile(r'"<em>'), lambda match: '<em>&laquo;&nbsp;'),
+        (re.compile(r'"<em>"</em><em>'), lambda match: '<em>&laquo;&nbsp;'),
+        (re.compile(r'"<a href='), lambda match: '&laquo;&nbsp;<a href='),
+        (re.compile(r'</em>"'), lambda match: '&nbsp;&raquo;</em>'),
+        (re.compile(r'</a>"'), lambda match: '&nbsp;&raquo;</a>'),
+        (re.compile(r'"</'), lambda match: '&nbsp;&raquo;</'),
+        (re.compile(r'>"'), lambda match: '>&laquo;&nbsp;'),
+        (re.compile(r'"<'), lambda match: '&nbsp;&raquo;<'),
+        (re.compile(r'&rsquo;"'), lambda match: '&rsquo;«&nbsp;'),
+        (re.compile(r' "'), lambda match: ' &laquo;&nbsp;'),
+        (re.compile(r'" '), lambda match: '&nbsp;&raquo; '),
+        (re.compile(r'"\.'), lambda match: '&nbsp;&raquo;.'),
+        (re.compile(r'",'), lambda match: '&nbsp;&raquo;,'),
+        (re.compile(r'"\?'), lambda match: '&nbsp;&raquo;?'),
+        (re.compile(r'":'), lambda match: '&nbsp;&raquo;:'),
+        (re.compile(r'";'), lambda match: '&nbsp;&raquo;;'),
+        (re.compile(r'"\!'), lambda match: '&nbsp;&raquo;!'),
+        (re.compile(r' :'), lambda match: '&nbsp;:'),
+        (re.compile(r' ;'), lambda match: '&nbsp;;'),
+        (re.compile(r' \?'), lambda match: '&nbsp;?'),
+        (re.compile(r' \!'), lambda match: '&nbsp;!'),
+        (re.compile(r'\s»'), lambda match: '&nbsp;»'),
+        (re.compile(r'«\s'), lambda match: '«&nbsp;'),
+        (re.compile(r' %'), lambda match: '&nbsp;%'),
+        (re.compile(r'\.jpg&nbsp;&raquo; width='), lambda match: '.jpg'),
+        (re.compile(r'\.png&nbsp;&raquo; width='), lambda match: '.png'),
+        (re.compile(r' &ndash; '), lambda match: '&nbsp;&ndash; '),
+        (re.compile(r'figcaption style="display:none"'), lambda match: 'figcaption'),
+        (re.compile(r' – '), lambda match: '&nbsp;&ndash; '),
+        (re.compile(r' - '), lambda match: '&nbsp;&ndash; '),
+        (re.compile(r' -,'), lambda match: '&nbsp;&ndash;,'),
+        (re.compile(r'&raquo;:'), lambda match: '&raquo;&nbsp;:'),
+        ]
+
+
+    keep_only_tags    = [
+                       dict(name='div', attrs={'class':['global']})
+                        ]
+
+    remove_tags = [
+                       dict(name='div', attrs={'class':['bloc_base meme_sujet']}),
+                       dict(name='p', attrs={'class':['lire']})
+                        ]
+
+    remove_tags_after = [dict(id='fb-like')]
+
+    def get_article_url(self, article):
+          url = article.get('guid', None)
+          if '/chat/' in url or '.blog' in url or '/video/' in url or '/sport/' in url or '/portfolio/' in url or '/visuel/' in url :
+              url = None
+          return url
+
+#    def get_article_url(self, article):
+#        link = article.get('link')
+#        if 'blog' not in link and ('chat' not in link):
+#             return link
 
     feeds          = [
                       ('A la une', 'http://www.lemonde.fr/rss/une.xml'),
@@ -66,11 +147,3 @@ class LeMonde(BasicNewsRecipe):
            cover_url = link_item.img['src']
 
         return cover_url
-
-    def get_article_url(self, article):
-        url = article.get('guid', None)
-        if '/chat/' in url or '.blog' in url or '/video/' in url or '/sport/' in url or '/portfolio/' in url or '/visuel/' in url :
-            url = None
-        return url
-
-

recipes/nrc_handelsblad.recipe

@@ -0,0 +1,76 @@
+__license__   = 'GPL v3'
+__copyright__ = '2012'
+'''
+nrc.nl
+'''
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class NRC(BasicNewsRecipe):
+    title                  = 'NRC Handelsblad'
+    __author__             = 'veezh'
+    description            = 'Nieuws (no subscription needed)'
+    oldest_article         = 1
+    max_articles_per_feed  = 100
+    no_stylesheets         = True
+    #delay                  = 1
+    use_embedded_content   = False
+    encoding               = 'utf-8'
+    publisher              = 'nrc.nl'
+    category               = 'news, Netherlands, world'
+    language               = 'nl'
+    timefmt = ''
+    #publication_type       = 'newsportal'
+    extra_css = '''
+                    h1{font-size:130%;}
+                    #h2{font-size:100%;font-weight:normal;}
+                    #.href{font-size:xx-small;}
+                    .bijschrift{color:#666666; font-size:x-small;}
+                    #.main-article-info{font-family:Arial,Helvetica,sans-serif;}
+                    #full-contents{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                    #match-stats-summary{font-size:small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                '''
+    #preprocess_regexps     = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
+    conversion_options = {
+                              'comments'        : description
+                             ,'tags'            : category
+                             ,'language'        : language
+                             ,'publisher'       : publisher
+                             ,'linearize_tables': True
+                          }
+
+    remove_empty_feeds = True
+
+    filterDuplicates = True
+
+    def preprocess_html(self, soup):
+        for alink in soup.findAll('a'):
+            if alink.string is not None:
+               tstr = alink.string
+               alink.replaceWith(tstr)
+        return soup
+
+    keep_only_tags = [dict(name='div', attrs={'class':'article'})]
+    remove_tags_after = [dict(id='broodtekst')]
+
+#    keep_only_tags    = [
+#                       dict(name='div', attrs={'class':['label']})
+#                        ]
+
+#    remove_tags_after = [dict(name='dl', attrs={'class':['tags']})]
+
+#    def get_article_url(self, article):
+#        link = article.get('link')
+#        if 'blog' not in link and ('chat' not in link):
+#             return link
+
+    feeds          = [
+#                      ('Nieuws', 'http://www.nrc.nl/rss.php'),
+                      ('Binnenland', 'http://www.nrc.nl/nieuws/categorie/binnenland/rss.php'),
+                      ('Buitenland', 'http://www.nrc.nl/nieuws/categorie/buitenland/rss.php'),
+                      ('Economie', 'http://www.nrc.nl/nieuws/categorie/economie/rss.php'),
+                      ('Wetenschap', 'http://www.nrc.nl/nieuws/categorie/wetenschap/rss.php'),
+                      ('Cultuur', 'http://www.nrc.nl/nieuws/categorie/cultuur/rss.php'),
+                      ('Boeken', 'http://www.nrc.nl/boeken/rss.php'),
+                      ('Tech', 'http://www.nrc.nl/tech/rss.php/'),
+                      ('Klimaat', 'http://www.nrc.nl/klimaat/rss.php/'),
+                      ]

recipes/ourdailybread.recipe

@@ -14,6 +14,7 @@ class OurDailyBread(BasicNewsRecipe):
     language              = 'en'
     max_articles_per_feed = 100
     no_stylesheets        = True
+    auto_cleanup = True
     use_embedded_content  = False
     category              = 'ODB, Daily Devotional, Bible, Christian Devotional, Devotional, RBC Ministries, Our Daily Bread, Devotionals, Daily Devotionals, Christian Devotionals, Faith, Bible Study, Bible Studies, Scripture, RBC, religion'
     encoding              = 'utf-8'
@@ -25,12 +26,12 @@ class OurDailyBread(BasicNewsRecipe):
                             ,'linearize_tables' : True
                          }
 
-    keep_only_tags    = [dict(attrs={'class':'module-content'})]
-    remove_tags       = [
-                           dict(attrs={'id':'article-zoom'})
-                          ,dict(attrs={'class':'listen-now-box'})
-                        ]
-    remove_tags_after = dict(attrs={'class':'readable-area'})
+    #keep_only_tags    = [dict(attrs={'class':'module-content'})]
+    #remove_tags       = [
+                           #dict(attrs={'id':'article-zoom'})
+                          #,dict(attrs={'class':'listen-now-box'})
+                        #]
+    #remove_tags_after = dict(attrs={'class':'readable-area'})
 
     extra_css = '''
                 .text{font-family:Arial,Helvetica,sans-serif;font-size:x-small;}

setup/installer/__init__.py

@@ -48,7 +48,7 @@ class Push(Command):
         threads = []
         for host in (
             r'Owner@winxp:/cygdrive/c/Documents\ and\ Settings/Owner/calibre',
-            'kovid@leopard_test:calibre',
+            'kovid@ox:calibre',
             r'kovid@win7:/cygdrive/c/Users/kovid/calibre',
             ):
             rcmd = BASE_RSYNC + EXCLUDES + ['.', host]

src/calibre/customize/builtins.py

@@ -1460,7 +1460,7 @@ class StoreNextoStore(StoreBase):
     actual_plugin = 'calibre.gui2.store.stores.nexto_plugin:NextoStore'
 
     headquarters = 'PL'
-    formats = ['EPUB', 'PDF']
+    formats = ['EPUB', 'MOBI', 'PDF']
     affiliate = True
 
 class StoreOpenBooksStore(StoreBase):

src/calibre/devices/android/driver.py

@@ -187,7 +187,7 @@ class ANDROID(USBMS):
             'UMS', '.K080', 'P990', 'LTE', 'MB853', 'GT-S5660_CARD', 'A107',
             'GT-I9003_CARD', 'XT912', 'FILE-CD_GADGET', 'RK29_SDK', 'MB855',
             'XT910', 'BOOK_A10', 'USB_2.0_DRIVER', 'I9100T', 'P999DW',
-            'KTABLET_PC', 'INGENIC']
+            'KTABLET_PC', 'INGENIC', 'GT-I9001_CARD']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',
@@ -195,7 +195,7 @@ class ANDROID(USBMS):
             'ANDROID_MID', 'P990_SD_CARD', '.K080', 'LTE_CARD', 'MB853',
             'A1-07___C0541A4F', 'XT912', 'MB855', 'XT910', 'BOOK_A10_CARD',
             'USB_2.0_DRIVER', 'I9100T', 'P999DW_SD_CARD', 'KTABLET_PC',
-            'FILE-CD_GADGET']
+            'FILE-CD_GADGET', 'GT-I9001_CARD']
 
     OSX_MAIN_MEM = 'Android Device Main Memory'
 

src/calibre/ebooks/conversion/utils.py

@@ -527,11 +527,17 @@ class HeuristicProcessor(object):
         if re.findall('(<|>)', replacement_break):
             if re.match('^<hr', replacement_break):
                 if replacement_break.find('width') != -1:
-                    width = int(re.sub('.*?width(:|=)(?P<wnum>\d+).*', '\g<wnum>', replacement_break))
-                    replacement_break = re.sub('(?i)(width=\d+\%?|width:\s*\d+(\%|px|pt|em)?;?)', '', replacement_break)
-                    divpercent = (100 - width) / 2
-                    hr_open = re.sub('45', str(divpercent), hr_open)
-                    scene_break = hr_open+replacement_break+'</div>'
+                    try:
+                        width = int(re.sub('.*?width(:|=)(?P<wnum>\d+).*', '\g<wnum>', replacement_break))
+                    except:
+                        scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
+                        self.log.warn('Invalid replacement scene break'
+                                ' expression, using default')
+                    else:
+                        replacement_break = re.sub('(?i)(width=\d+\%?|width:\s*\d+(\%|px|pt|em)?;?)', '', replacement_break)
+                        divpercent = (100 - width) / 2
+                        hr_open = re.sub('45', str(divpercent), hr_open)
+                        scene_break = hr_open+replacement_break+'</div>'
                 else:
                     scene_break = hr_open+'<hr style="height: 3px; background:#505050" /></div>'
             elif re.match('^<img', replacement_break):

src/calibre/ebooks/mobi/debug/headers.py

@@ -205,7 +205,10 @@ class EXTHHeader(object):
 
     @property
     def kf8_header_index(self):
-        return self.get(121, None)
+        ans = self.get(121, None)
+        if ans == NULL_INDEX:
+            ans = None
+        return ans
 
     def __str__(self):
         ans = ['*'*20 + ' EXTH Header '+ '*'*20]
@@ -467,9 +470,15 @@ class MOBIFile(object):
         if mh.file_version >= 8:
             self.kf8_type = 'standalone'
         elif mh.has_exth and mh.exth.kf8_header_index is not None:
-            self.kf8_type = 'joint'
             kf8i = mh.exth.kf8_header_index
-            mh8 = MOBIHeader(self.records[kf8i], kf8i)
+            try:
+                rec = self.records[kf8i-1]
+            except IndexError:
+                pass
+            else:
+                if rec.raw == b'BOUNDARY':
+                    self.kf8_type = 'joint'
+                    mh8 = MOBIHeader(self.records[kf8i], kf8i)
         self.mobi8_header = mh8
 
         if 'huff' in self.mobi_header.compression.lower():

src/calibre/ebooks/mobi/debug/mobi8.py

@@ -7,9 +7,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os
+import sys, os, imghdr
 
 from calibre.ebooks.mobi.debug.headers import TextRecord
+from calibre.ebooks.mobi.utils import read_font_record
 
 class MOBIFile(object):
 
@@ -30,6 +31,7 @@ class MOBIFile(object):
                 first_text_record+offset+h8.number_of_text_records])]
 
         self.raw_text = b''.join(r.raw for r in self.text_records)
+        self.extract_resources()
 
     def print_header(self, f=sys.stdout):
         print (str(self.mf.palmdb).encode('utf-8'), file=f)
@@ -41,6 +43,42 @@ class MOBIFile(object):
         print (file=f)
         print (str(self.mf.mobi8_header).encode('utf-8'), file=f)
 
+    def extract_resources(self):
+        self.resource_map = []
+        known_types = {b'FLIS', b'FCIS', b'SRCS',
+                    b'\xe9\x8e\r\n', b'RESC', b'BOUN', b'FDST', b'DATP',
+                    b'AUDI', b'VIDE'}
+
+        for i, rec in enumerate(self.resource_records):
+            sig = rec.raw[:4]
+            payload = rec.raw
+            ext = 'dat'
+            prefix = 'binary'
+            suffix = ''
+            if sig in {b'HUFF', b'CDIC', b'INDX'}: continue
+            # TODO: Ignore CNCX records as well
+            if sig == b'FONT':
+                font = read_font_record(rec.raw)
+                if font['err']:
+                    raise ValueError('Failed to read font record: %s Headers: %s'%(
+                        font['err'], font['headers']))
+                payload = (font['font_data'] if font['font_data'] else
+                        font['raw_data'])
+                prefix, ext = 'fonts', font['ext']
+            elif sig not in known_types:
+                q = imghdr.what(None, rec.raw)
+                if q:
+                    prefix, ext = 'images', q
+
+            if prefix == 'binary':
+                if sig == b'\xe9\x8e\r\n':
+                    suffix = '-EOF'
+                elif sig in known_types:
+                    suffix = '-' + sig.decode('ascii')
+
+            self.resource_map.append(('%s/%06d%s.%s'%(prefix, i, suffix, ext),
+                payload))
+
 
 def inspect_mobi(mobi_file, ddir):
     f = MOBIFile(mobi_file)
@@ -51,12 +89,14 @@ def inspect_mobi(mobi_file, ddir):
     with open(alltext, 'wb') as of:
         of.write(f.raw_text)
 
-    for tdir, attr in [('text_records', 'text_records'), ('images',
-        'image_records'), ('binary', 'binary_records'), ('font',
-            'font_records')]:
-        tdir = os.path.join(ddir, tdir)
-        os.mkdir(tdir)
-        for rec in getattr(f, attr, []):
-            rec.dump(tdir)
+    for x in ('text_records', 'images', 'fonts', 'binary'):
+        os.mkdir(os.path.join(ddir, x))
+
+    for rec in f.text_records:
+        rec.dump(os.path.join(ddir, 'text_records'))
+
+    for href, payload in f.resource_map:
+        with open(os.path.join(ddir, href), 'wb') as f:
+            f.write(payload)
 
 

src/calibre/ebooks/mobi/reader/headers.py

@@ -11,7 +11,7 @@ import struct, re, os
 from calibre import replace_entities
 from calibre.utils.date import parse_date
 from calibre.ebooks.mobi import MobiError
-from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.mobi.langcodes import main_language, sub_language, mobi2iana
 
 NULL_INDEX = 0xffffffff
@@ -75,10 +75,14 @@ class EXTHHeader(object): # {{{
                 self.mi.author_sort = au.strip()
         elif idx == 101:
             self.mi.publisher = content.decode(codec, 'ignore').strip()
+            if self.mi.publisher in {'Unknown', _('Unknown')}:
+                self.mi.publisher = None
         elif idx == 103:
             self.mi.comments  = content.decode(codec, 'ignore')
         elif idx == 104:
-            self.mi.isbn      = content.decode(codec, 'ignore').strip().replace('-', '')
+            raw = check_isbn(content.decode(codec, 'ignore').strip().replace('-', ''))
+            if raw:
+                self.mi.isbn = raw
         elif idx == 105:
             if not self.mi.tags:
                 self.mi.tags = []
@@ -92,12 +96,24 @@ class EXTHHeader(object): # {{{
                 pass
         elif idx == 108:
             self.mi.book_producer = content.decode(codec, 'ignore').strip()
+        elif idx == 112: # dc:source set in some EBSP amazon samples
+            try:
+                content = content.decode(codec).strip()
+                isig = 'urn:isbn:'
+                if content.lower().startswith(isig):
+                    raw = check_isbn(content[len(isig):])
+                    if raw and not self.mi.isbn:
+                        self.mi.isbn = raw
+            except:
+                pass
         elif idx == 113:
             pass # ASIN or UUID
         elif idx == 116:
             self.start_offset, = struct.unpack(b'>L', content)
         elif idx == 121:
             self.kf8_header, = struct.unpack(b'>L', content)
+            if self.kf8_header == NULL_INDEX:
+                self.kf8_header = None
         #else:
         #    print 'unhandled metadata record', idx, repr(content)
 # }}}

src/calibre/ebooks/mobi/reader/index.py

@@ -39,10 +39,41 @@ def parse_indx_header(data):
     words = (
             'len', 'nul1', 'type', 'gen', 'start', 'count', 'code',
             'lng', 'total', 'ordt', 'ligt', 'nligt', 'ncncx'
-    )
+    ) + tuple('unknown%d'%i for i in xrange(27)) + ('ocnt', 'oentries',
+            'ordt1', 'ordt2', 'tagx')
     num = len(words)
     values = struct.unpack(bytes('>%dL' % num), data[4:4*(num+1)])
-    return dict(zip(words, values))
+    ans = dict(zip(words, values))
+    ordt1, ordt2 = ans['ordt1'], ans['ordt2']
+    ans['ordt1_raw'], ans['ordt2_raw'] = [], []
+    ans['ordt_map'] = ''
+
+    if ordt1 > 0 and data[ordt1:ordt1+4] == b'ORDT':
+        # I dont know what this is, but using it seems to be unnecessary, so
+        # just leave it as the raw bytestring
+        ans['ordt1_raw'] = data[ordt1+4:ordt1+4+ans['oentries']]
+    if ordt2 > 0 and data[ordt2:ordt2+4] == b'ORDT':
+        ans['ordt2_raw'] = raw = bytearray(data[ordt2+4:ordt2+4+2*ans['oentries']])
+        if ans['code'] == 65002:
+            # This appears to be EBCDIC-UTF (65002) encoded. I can't be
+            # bothered to write a decoder for this (see
+            # http://www.unicode.org/reports/tr16/) Just how stupid is Amazon?
+            # Instead, we use a weird hack that seems to do the trick for all
+            # the books with this type of ORDT record that I have come across.
+            # Some EBSP book samples in KF8 format from Amazon have this type
+            # of encoding.
+            # Basically we try to interpret every second byte as a printable
+            # ascii character. If we cannot, we map to the ? char.
+
+            parsed = bytearray(ans['oentries'])
+            for i in xrange(0, 2*ans['oentries'], 2):
+                parsed[i//2] = raw[i+1] if 0x20 < raw[i+1] < 0x7f else ord(b'?')
+            ans['ordt_map'] = bytes(parsed).decode('ascii')
+        else:
+            ans['ordt_map'] = '?'*ans['oentries']
+
+    return ans
+
 
 class CNCX(object): # {{{
 
@@ -163,7 +194,7 @@ def get_tag_map(control_byte_count, tagx, data, strict=False):
     return ans
 
 def parse_index_record(table, data, control_byte_count, tags, codec,
-        strict=False):
+        ordt_map, strict=False):
     header = parse_indx_header(data)
     idxt_pos = header['start']
     if data[idxt_pos:idxt_pos+4] != b'IDXT':
@@ -184,12 +215,11 @@ def parse_index_record(table, data, control_byte_count, tags, codec,
     for j in xrange(entry_count):
         start, end = idx_positions[j:j+2]
         rec = data[start:end]
-        ident, consumed = decode_string(rec, codec=codec)
+        ident, consumed = decode_string(rec, codec=codec, ordt_map=ordt_map)
         rec = rec[consumed:]
         tag_map = get_tag_map(control_byte_count, tags, rec, strict=strict)
         table[ident] = tag_map
 
-
 def read_index(sections, idx, codec):
     table, cncx = OrderedDict(), CNCX([], codec)
 
@@ -203,12 +233,13 @@ def read_index(sections, idx, codec):
         cncx_records = [x[0] for x in sections[off:off+indx_header['ncncx']]]
         cncx = CNCX(cncx_records, codec)
 
-    tag_section_start = indx_header['len']
+    tag_section_start = indx_header['tagx']
     control_byte_count, tags = parse_tagx_section(data[tag_section_start:])
 
     for i in xrange(idx + 1, idx + 1 + indx_count):
         # Index record
         data = sections[i][0]
-        parse_index_record(table, data, control_byte_count, tags, codec)
+        parse_index_record(table, data, control_byte_count, tags, codec,
+                indx_header['ordt_map'])
     return table, cncx
 

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -285,7 +285,11 @@ class Mobi8Reader(object):
     def create_guide(self):
         guide = Guide()
         for ref_type, ref_title, fileno in self.guide:
-            elem = self.elems[fileno]
+            try:
+                elem = self.elems[fileno]
+            except IndexError:
+                # Happens for thumbnailstandard in Amazon book samples
+                continue
             fi = self.get_file_info(elem.insert_pos)
             idtext = self.get_id_tag(elem.insert_pos).decode(self.header.codec)
             linktgt = fi.filename

src/calibre/ebooks/mobi/utils.py

@@ -15,10 +15,12 @@ from calibre.ebooks import normalize
 
 IMAGE_MAX_SIZE = 10 * 1024 * 1024
 
-def decode_string(raw, codec='utf-8'):
+def decode_string(raw, codec='utf-8', ordt_map=''):
     length, = struct.unpack(b'>B', raw[0])
     raw = raw[1:1+length]
     consumed = length+1
+    if ordt_map:
+        return ''.join(ordt_map[ord(x)] for x in raw), consumed
     return raw.decode(codec), consumed
 
 def decode_hex_number(raw, codec='utf-8'):

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -161,8 +161,8 @@ class Serializer(object):
                 self.serialize_text(ref.title, quot=True)
                 buf.write(b'" ')
                 if (ref.title.lower() == 'start' or
-                    (ref.type and ref.type.lower() in ('start',
-                        'other.start'))):
+                    (ref.type and ref.type.lower() in {'start',
+                        'other.start', 'text'})):
                     self._start_href = ref.href
             self.serialize_href(ref.href)
             # Space required or won't work, I kid you not

src/calibre/ebooks/oeb/display/__init__.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/ebooks/oeb/display/webview.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+from calibre import guess_type
+
+class EntityDeclarationProcessor(object): # {{{
+
+    def __init__(self, html):
+        self.declared_entities = {}
+        for match in re.finditer(r'<!\s*ENTITY\s+([^>]+)>', html):
+            tokens = match.group(1).split()
+            if len(tokens) > 1:
+                self.declared_entities[tokens[0].strip()] = tokens[1].strip().replace('"', '')
+        self.processed_html = html
+        for key, val in self.declared_entities.iteritems():
+            self.processed_html = self.processed_html.replace('&%s;'%key, val)
+# }}}
+
+def self_closing_sub(match):
+    tag = match.group(1)
+    if tag.lower().strip() == 'br':
+        return match.group()
+    return '<%s %s></%s>'%(match.group(1), match.group(2), match.group(1))
+
+def load_html(path, view, codec='utf-8', mime_type=None,
+        pre_load_callback=lambda x:None):
+    from PyQt4.Qt import QUrl, QByteArray
+    if mime_type is None:
+        mime_type = guess_type(path)[0]
+    with open(path, 'rb') as f:
+        html = f.read().decode(codec, 'replace')
+
+    html = EntityDeclarationProcessor(html).processed_html
+    has_svg = re.search(r'<[:a-zA-Z]*svg', html) is not None
+    if 'xhtml' in mime_type:
+        self_closing_pat = re.compile(r'<([a-z1-6]+)\s+([^>]+)/>',
+                re.IGNORECASE)
+        html = self_closing_pat.sub(self_closing_sub, html)
+
+    html = re.sub(ur'<\s*title\s*/\s*>', u'', html, flags=re.IGNORECASE)
+    loading_url = QUrl.fromLocalFile(path)
+    pre_load_callback(loading_url)
+
+    if has_svg:
+        view.setContent(QByteArray(html.encode(codec)), mime_type,
+                loading_url)
+    else:
+        view.setHtml(html, loading_url)
+
+
+

src/calibre/ebooks/oeb/iterator.py

@@ -26,6 +26,8 @@ from calibre.constants import filesystem_encoding
 TITLEPAGE = CoverManager.SVG_TEMPLATE.decode('utf-8').replace(\
         '__ar__', 'none').replace('__viewbox__', '0 0 600 800'
         ).replace('__width__', '600').replace('__height__', '800')
+BM_FIELD_SEP = u'*|!|?|*'
+BM_LEGACY_ESC = u'esc-text-%&*#%(){}ads19-end-esc'
 
 def character_count(html):
     '''
@@ -273,27 +275,62 @@ class EbookIterator(object):
 
     def parse_bookmarks(self, raw):
         for line in raw.splitlines():
+            bm = None
             if line.count('^') > 0:
                 tokens = line.rpartition('^')
                 title, ref = tokens[0], tokens[2]
-                self.bookmarks.append((title, ref))
+                try:
+                    spine, _, pos = ref.partition('#')
+                    spine = int(spine.strip())
+                except:
+                    continue
+                bm = {'type':'legacy', 'title':title, 'spine':spine, 'pos':pos}
+            elif BM_FIELD_SEP in line:
+                try:
+                    title, spine, pos = line.strip().split(BM_FIELD_SEP)
+                    spine = int(spine)
+                except:
+                    continue
+                # Unescape from serialization
+                pos = pos.replace(BM_LEGACY_ESC, u'^')
+                # Check for pos being a scroll fraction
+                try:
+                    pos = float(pos)
+                except:
+                    pass
+                bm = {'type':'cfi', 'title':title, 'pos':pos, 'spine':spine}
+
+            if bm:
+                self.bookmarks.append(bm)
 
     def serialize_bookmarks(self, bookmarks):
         dat = []
-        for title, bm in bookmarks:
-            dat.append(u'%s^%s'%(title, bm))
-        return (u'\n'.join(dat) +'\n').encode('utf-8')
+        for bm in bookmarks:
+            if bm['type'] == 'legacy':
+                rec = u'%s^%d#%s'%(bm['title'], bm['spine'], bm['pos'])
+            else:
+                pos = bm['pos']
+                if isinstance(pos, (int, float)):
+                    pos = unicode(pos)
+                else:
+                    pos = pos.replace(u'^', BM_LEGACY_ESC)
+                rec = BM_FIELD_SEP.join([bm['title'], unicode(bm['spine']), pos])
+            dat.append(rec)
+        return (u'\n'.join(dat) +u'\n')
 
     def read_bookmarks(self):
         self.bookmarks = []
         bmfile = os.path.join(self.base, 'META-INF', 'calibre_bookmarks.txt')
         raw = ''
         if os.path.exists(bmfile):
-            raw = open(bmfile, 'rb').read().decode('utf-8')
+            with open(bmfile, 'rb') as f:
+                raw = f.read()
         else:
             saved = self.config['bookmarks_'+self.pathtoebook]
             if saved:
                 raw = saved
+        if not isinstance(raw, unicode):
+            raw = raw.decode('utf-8')
         self.parse_bookmarks(raw)
 
     def save_bookmarks(self, bookmarks=None):
@@ -306,18 +343,15 @@ class EbookIterator(object):
                 zf = open(self.pathtoebook, 'r+b')
             except IOError:
                 return
-            safe_replace(zf, 'META-INF/calibre_bookmarks.txt', StringIO(dat),
+            safe_replace(zf, 'META-INF/calibre_bookmarks.txt',
+                    StringIO(dat.encode('utf-8')),
                     add_missing=True)
         else:
             self.config['bookmarks_'+self.pathtoebook] = dat
 
     def add_bookmark(self, bm):
-        dups = []
-        for x in self.bookmarks:
-            if x[0] == bm[0]:
-                dups.append(x)
-        for x in dups:
-            self.bookmarks.remove(x)
+        self.bookmarks = [x for x in self.bookmarks if x['title'] !=
+                bm['title']]
         self.bookmarks.append(bm)
         self.save_bookmarks()
 

src/calibre/ebooks/oeb/transforms/guide.py

@@ -8,10 +8,9 @@ __docformat__ = 'restructuredtext en'
 
 
 class Clean(object):
-    '''Clean up guide, leaving only a pointer to the cover'''
+    '''Clean up guide, leaving only known values '''
 
     def __call__(self, oeb, opts):
-        from calibre.ebooks.oeb.base import urldefrag
         self.oeb, self.log, self.opts = oeb, oeb.log, opts
 
         if 'cover' not in self.oeb.guide:
@@ -32,10 +31,15 @@ class Clean(object):
                 ref.type = 'cover'
                 self.oeb.guide.refs['cover'] = ref
 
+        if ('start' in self.oeb.guide and 'text' not in self.oeb.guide):
+            # Prefer text to start as per the OPF 2.0 spec
+            x = self.oeb.guide['start']
+            self.oeb.guide.add('text', x.title, x.href)
+            self.oeb.guide.remove('start')
+
         for x in list(self.oeb.guide):
-            href = urldefrag(self.oeb.guide[x].href)[0]
-            if x.lower() not in ('cover', 'titlepage', 'masthead', 'toc',
-                    'title-page', 'copyright-page', 'start'):
+            if x.lower() not in {'cover', 'titlepage', 'masthead', 'toc',
+                    'title-page', 'copyright-page', 'text'}:
                 item = self.oeb.guide[x]
                 if item.title and item.title.lower() == 'start':
                     continue

src/calibre/ebooks/pdf/writer.py

@@ -18,10 +18,11 @@ from calibre.ebooks.pdf.pageoptions import unit, paper_size, \
 from calibre.ebooks.metadata import authors_to_string
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre import __appname__, __version__, fit_image
+from calibre.ebooks.oeb.display.webview import load_html
 
 from PyQt4 import QtCore
-from PyQt4.Qt import QUrl, QEventLoop, QObject, \
-    QPrinter, QMetaObject, QSizeF, Qt, QPainter, QPixmap
+from PyQt4.Qt import (QEventLoop, QObject,
+    QPrinter, QMetaObject, QSizeF, Qt, QPainter, QPixmap)
 from PyQt4.QtWebKit import QWebView
 
 from pyPdf import PdfFileWriter, PdfFileReader
@@ -70,7 +71,7 @@ def get_pdf_printer(opts, for_comic=False):
                 opts.margin_right, opts.margin_bottom, QPrinter.Point)
     printer.setOrientation(orientation(opts.orientation))
     printer.setOutputFormat(QPrinter.PdfFormat)
-    printer.setFullPage(True)
+    printer.setFullPage(for_comic)
     return printer
 
 def get_printer_page_size(opts, for_comic=False):
@@ -156,8 +157,7 @@ class PDFWriter(QObject): # {{{
         self.combine_queue.append(os.path.join(self.tmp_path, '%i.pdf' % (len(self.combine_queue) + 1)))
 
         self.logger.debug('Processing %s...' % item)
-
-        self.view.load(QUrl.fromLocalFile(item))
+        load_html(item, self.view)
 
     def _render_html(self, ok):
         if ok:
@@ -168,9 +168,14 @@ class PDFWriter(QObject): # {{{
             # We have to set the engine to Native on OS X after the call to set
             # filename. Setting a filename with .pdf as the extension causes
             # Qt to set the format to use Qt's PDF engine even if native was
-            # previously set on the printer.
+            # previously set on the printer. Qt's PDF engine produces image
+            # based PDFs on OS X, so we cannot use it.
             if isosx:
                 printer.setOutputFormat(QPrinter.NativeFormat)
+            self.view.page().mainFrame().evaluateJavaScript('''
+                document.body.style.backgroundColor = "white";
+
+                ''')
             self.view.print_(printer)
             printer.abort()
         else:

src/calibre/gui2/actions/copy_to_library.py

@@ -81,8 +81,8 @@ class Worker(Thread): # {{{
             if prefs['add_formats_to_existing']:
                 identical_book_list = newdb.find_identical_books(mi)
                 if identical_book_list: # books with same author and nearly same title exist in newdb
-                    self.auto_merged_ids[x] = _('%s by %s')%(mi.title,
-                            mi.format_field('authors')[1])
+                    self.auto_merged_ids[x] = _('%(title)s by %(author)s')%\
+                    dict(title=mi.title, author=mi.format_field('authors')[1])
                     automerged = True
                     seen_fmts = set()
                     for identical_book in identical_book_list:

src/calibre/gui2/dialogs/choose_plugin_toolbars.py

@@ -9,8 +9,8 @@ __docformat__ = 'restructuredtext en'
 __license__   = 'GPL v3'
 
 
-from PyQt4.Qt import QDialog, QVBoxLayout, QLabel, QDialogButtonBox,  \
-            QListWidget, QAbstractItemView
+from PyQt4.Qt import (QDialog, QVBoxLayout, QLabel, QDialogButtonBox,
+            QListWidget, QAbstractItemView)
 from PyQt4 import QtGui
 
 class ChoosePluginToolbarsDialog(QDialog):
@@ -39,6 +39,9 @@ class ChoosePluginToolbarsDialog(QDialog):
         self._locations_list.setSizePolicy(sizePolicy)
         for key, text in locations:
             self._locations_list.addItem(text)
+            if key in {'toolbar', 'toolbar-device'}:
+                self._locations_list.item(self._locations_list.count()-1
+                        ).setSelected(True)
         self._layout.addWidget(self._locations_list)
 
         self._footer_label = QLabel(

src/calibre/gui2/dialogs/scheduler.py

@@ -11,9 +11,9 @@ from datetime import timedelta
 import calendar, textwrap
 from collections import OrderedDict
 
-from PyQt4.Qt import QDialog, Qt, QTime, QObject, QMenu, QHBoxLayout, \
-        QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QGridLayout, \
-        QCheckBox, QTimeEdit, QLabel, QLineEdit, QDoubleSpinBox
+from PyQt4.Qt import (QDialog, Qt, QTime, QObject, QMenu, QHBoxLayout,
+        QAction, QIcon, QMutex, QTimer, pyqtSignal, QWidget, QGridLayout,
+        QCheckBox, QTimeEdit, QLabel, QLineEdit, QDoubleSpinBox)
 
 from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
 from calibre.gui2 import config as gconf, error_dialog
@@ -317,6 +317,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                     return False
             if un or pw:
                 self.recipe_model.set_account_info(urn, un, pw)
+            else:
+                self.recipe_model.clear_account_info(urn)
 
         if self.schedule.isChecked():
             schedule_type, schedule = \

src/calibre/gui2/store/search_result.py

@@ -29,4 +29,4 @@ class SearchResult(object):
         self.plugin_author = ''
 
     def __eq__(self, other):
-        return self.title == other.title and self.author == other.author and self.store_name == other.store_name
+        return self.title == other.title and self.author == other.author and self.store_name == other.store_name and self.formats == other.formats

src/calibre/gui2/store/stores/ebookpoint_plugin.py

@@ -3,7 +3,7 @@
 from __future__ import (unicode_literals, division, absolute_import, print_function)
 
 __license__ = 'GPL 3'
-__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>'
+__copyright__ = '2011-2012, Tomasz Długosz <tomek3d@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import re
@@ -64,9 +64,7 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
                 author = ''.join(data.xpath('.//p[@class="author"]/text()'))
                 price = ''.join(data.xpath('.//p[@class="price"]/ins/text()'))
 
-                with closing(br.open(id.strip(), timeout=timeout)) as nf:
-                    idata = html.fromstring(nf.read())
-                    formats = ', '.join(idata.xpath('//dd[@class="radio-line"]/label/text()'))
+                formats = ', '.join(data.xpath('.//div[@class="ikony"]/span/text()'))
 
                 counter -= 1
 
@@ -77,6 +75,6 @@ class EbookpointStore(BasicStoreConfig, StorePlugin):
                 s.price = re.sub(r'\.',',',price)
                 s.detail_item = id.strip()
                 s.drm = SearchResult.DRM_UNLOCKED
-                s.formats = formats.upper().strip()
+                s.formats = formats.upper()
 
                 yield s

src/calibre/gui2/store/stores/nexto_plugin.py

@@ -68,8 +68,8 @@ class NextoStore(BasicStoreConfig, StorePlugin):
                     title = ''.join(data.xpath('.//a[@class="title"]/text()'))
                     title = re.sub(r' - ebook$', '', title)
                     formats = ', '.join(data.xpath('.//ul[@class="formats_available"]/li//b/text()'))
-                    DrmFree = re.search(r'bez.DRM', formats)
-                    formats = re.sub(r'\(.+\)', '', formats)
+                    DrmFree = re.search(r'znak', formats)
+                    formats = re.sub(r'\ ?\(.+?\)', '', formats)
 
                     author = ''
                     with closing(br.open('http://www.nexto.pl/' + id.strip(), timeout=timeout/4)) as nf:

src/calibre/gui2/store/stores/woblink_plugin.py

@@ -6,6 +6,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2011-2012, Tomasz Długosz <tomek3d@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
+import copy
 import re
 import urllib
 from contextlib import closing
@@ -43,9 +44,9 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
         url = 'http://woblink.com/publication?query=' + urllib.quote_plus(query.encode('utf-8'))
         if max_results > 10:
             if max_results > 20:
-                url += '&limit=' + str(30)
+                url += '&limit=30'
             else:
-                url += '&limit=' + str(20)
+                url += '&limit=20'
 
         br = browser()
 
@@ -66,15 +67,6 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
                 price = ''.join(data.xpath('.//div[@class="prices"]/span[1]/span/text()'))
                 price = re.sub('\.', ',', price)
                 formats = [ form[8:-4].split('_')[0] for form in data.xpath('.//p[3]/img/@src')]
-                if 'epub' in formats:
-                    formats.remove('epub')
-                    formats.append('WOBLINK')
-                    if 'E Ink' in data.xpath('.//div[@class="prices"]/img/@title'):
-                        formats.insert(0, 'EPUB')
-                if 'pdf' in formats:
-                    formats[formats.index('pdf')] = 'PDF' 
-
-                counter -= 1
 
                 s = SearchResult()
                 s.cover_url = 'http://woblink.com' + cover_url
@@ -82,7 +74,28 @@ class WoblinkStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price + ' zł'
                 s.detail_item = id.strip()
-                s.drm = SearchResult.DRM_UNKNOWN if 'MOBI' in formats else SearchResult.DRM_LOCKED
-                s.formats = ', '.join(formats)
-
-                yield s
+                
+                # MOBI should be send first,
+                if 'MOBI' in formats:
+                    t = copy.copy(s)
+                    t.title += ' MOBI'
+                    t.drm = SearchResult.DRM_UNLOCKED
+                    t.formats = 'MOBI'
+                    formats.remove('MOBI')
+                    
+                    counter -= 1
+                    yield t
+                    
+                # and the remaining formats (if any) next
+                if formats:
+                    if 'epub' in formats:
+                        formats.remove('epub')
+                        formats.append('WOBLINK')
+                        if 'E Ink' in data.xpath('.//div[@class="prices"]/img/@title'):
+                            formats.insert(0, 'EPUB')
+                    
+                    s.drm = SearchResult.DRM_LOCKED
+                    s.formats = ', '.join(formats).upper()
+                    
+                    counter -= 1
+                    yield s

src/calibre/gui2/update.py

@@ -151,7 +151,7 @@ class UpdateMixin(object):
             plt = u''
             if has_plugin_updates:
                 plt = _(' (%d plugin updates)')%plugin_updates
-            msg = (u'<span style="color:red; font-weight: bold">%s: '
+            msg = (u'<span style="color:green; font-weight: bold">%s: '
                     u'<a href="update:%s">%s%s</a></span>') % (
                         _('Update found'), version, calibre_version, plt)
         else:

src/calibre/gui2/viewer/bookmarkmanager.py

@@ -31,6 +31,7 @@ class BookmarkManager(QDialog, Ui_BookmarkManager):
             bookmarks = self.bookmarks[:]
         self._model = BookmarkTableModel(self, bookmarks)
         self.bookmarks_table.setModel(self._model)
+        self.bookmarks_table.resizeColumnsToContents()
 
     def delete_bookmark(self):
         indexes = self.bookmarks_table.selectionModel().selectedIndexes()
@@ -80,7 +81,7 @@ class BookmarkManager(QDialog, Ui_BookmarkManager):
             if not bad:
                 bookmarks = self._model.bookmarks[:]
                 for bm in imported:
-                    if bm not in bookmarks and bm[0] != 'calibre_current_page_bookmark':
+                    if bm not in bookmarks and bm['title'] != 'calibre_current_page_bookmark':
                         bookmarks.append(bm)
                 self.set_bookmarks(bookmarks)
 
@@ -105,13 +106,14 @@ class BookmarkTableModel(QAbstractTableModel):
 
     def data(self, index, role):
         if role in (Qt.DisplayRole, Qt.EditRole):
-            ans = self.bookmarks[index.row()][0]
+            ans = self.bookmarks[index.row()]['title']
             return NONE if ans is None else QVariant(ans)
         return NONE
 
     def setData(self, index, value, role):
         if role == Qt.EditRole:
-            self.bookmarks[index.row()] = (unicode(value.toString()).strip(), self.bookmarks[index.row()][1])
+            bm = self.bookmarks[index.row()]
+            bm['title'] = unicode(value.toString()).strip()
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), index, index)
             return True
         return False

src/calibre/gui2/viewer/documentview.py

@@ -4,14 +4,14 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
 # Imports {{{
-import os, math, re, glob, sys, zipfile
+import os, math, glob, zipfile
 from base64 import b64encode
 from functools import partial
 
 from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt,
                      QPainter, QPalette, QBrush, QFontDatabase, QDialog,
                      QColor, QPoint, QImage, QRegion, QVariant, QIcon,
-                     QFont, pyqtSignature, QAction, QByteArray, QMenu,
+                     QFont, pyqtSignature, QAction, QMenu,
                      pyqtSignal, QSwipeGesture, QApplication)
 from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings
 
@@ -21,10 +21,11 @@ from calibre.gui2.viewer.config_ui import Ui_Dialog
 from calibre.gui2.viewer.flip import SlideFlip
 from calibre.gui2.shortcuts import Shortcuts, ShortcutConfig
 from calibre.constants import iswindows
-from calibre import prints, guess_type
+from calibre import prints
 from calibre.gui2.viewer.keys import SHORTCUTS
 from calibre.gui2.viewer.javascript import JavaScriptLoader
 from calibre.gui2.viewer.position import PagePosition
+from calibre.ebooks.oeb.display.webview import load_html
 
 # }}}
 
@@ -312,10 +313,14 @@ class Document(QWebPage): # {{{
         self.javascript('goto_reference("%s")'%ref)
 
     def goto_bookmark(self, bm):
-        bm = bm.strip()
-        if bm.startswith('>'):
-            bm = bm[1:].strip()
-        self.javascript('scroll_to_bookmark("%s")'%bm)
+        if bm['type'] == 'legacy':
+            bm = bm['pos']
+            bm = bm.strip()
+            if bm.startswith('>'):
+                bm = bm[1:].strip()
+            self.javascript('scroll_to_bookmark("%s")'%bm)
+        elif bm['type'] == 'cfi':
+            self.page_position.to_pos(bm['pos'])
 
     def javascript(self, string, typ=None):
         ans = self.mainFrame().evaluateJavaScript(string)
@@ -366,40 +371,9 @@ class Document(QWebPage): # {{{
     def elem_outer_xml(self, elem):
         return unicode(elem.toOuterXml())
 
-    def find_bookmark_element(self):
-        mf = self.mainFrame()
-        doc_pos = self.ypos
-        min_delta, min_elem = sys.maxint, None
-        for y in range(10, -500, -10):
-            for x in range(-50, 500, 10):
-                pos = QPoint(x, y)
-                result = mf.hitTestContent(pos)
-                if result.isNull(): continue
-                elem = result.enclosingBlockElement()
-                if elem.isNull(): continue
-                try:
-                    ypos = self.element_ypos(elem)
-                except:
-                    continue
-                delta = abs(ypos - doc_pos)
-                if delta < 25:
-                    return elem
-                if delta < min_delta:
-                    min_elem, min_delta = elem, delta
-        return min_elem
-
-
     def bookmark(self):
-        elem = self.find_bookmark_element()
-
-        if elem is None or self.element_ypos(elem) < 100:
-            bm = 'body|%f'%(float(self.ypos)/(self.height*0.7))
-        else:
-            bm = unicode(elem.evaluateJavaScript(
-                'calculate_bookmark(%d, this)'%self.ypos).toString())
-            if not bm:
-                bm = 'body|%f'%(float(self.ypos)/(self.height*0.7))
-        return bm
+        pos = self.page_position.current_pos
+        return {'type':'cfi', 'pos':pos}
 
     @property
     def at_bottom(self):
@@ -474,19 +448,6 @@ class Document(QWebPage): # {{{
 
 # }}}
 
-class EntityDeclarationProcessor(object): # {{{
-
-    def __init__(self, html):
-        self.declared_entities = {}
-        for match in re.finditer(r'<!\s*ENTITY\s+([^>]+)>', html):
-            tokens = match.group(1).split()
-            if len(tokens) > 1:
-                self.declared_entities[tokens[0].strip()] = tokens[1].strip().replace('"', '')
-        self.processed_html = html
-        for key, val in self.declared_entities.iteritems():
-            self.processed_html = self.processed_html.replace('&%s;'%key, val)
-# }}}
-
 class DocumentView(QWebView): # {{{
 
     magnification_changed = pyqtSignal(object)
@@ -497,8 +458,6 @@ class DocumentView(QWebView): # {{{
         self.is_auto_repeat_event = False
         self.debug_javascript = debug_javascript
         self.shortcuts =  Shortcuts(SHORTCUTS, 'shortcuts/viewer')
-        self.self_closing_pat = re.compile(r'<([a-z1-6]+)\s+([^>]+)/>',
-                re.IGNORECASE)
         self.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding))
         self._size_hint = QSize(510, 680)
         self.initial_pos = 0.0
@@ -689,31 +648,16 @@ class DocumentView(QWebView): # {{{
     def path(self):
         return os.path.abspath(unicode(self.url().toLocalFile()))
 
-    def self_closing_sub(self, match):
-        tag = match.group(1)
-        if tag.lower().strip() == 'br':
-            return match.group()
-        return '<%s %s></%s>'%(match.group(1), match.group(2), match.group(1))
-
     def load_path(self, path, pos=0.0):
         self.initial_pos = pos
-        mt = getattr(path, 'mime_type', None)
-        if mt is None:
-            mt = guess_type(path)[0]
-        html = open(path, 'rb').read().decode(path.encoding, 'replace')
-        html = EntityDeclarationProcessor(html).processed_html
-        has_svg = re.search(r'<[:a-zA-Z]*svg', html) is not None
 
-        if 'xhtml' in mt:
-            html = self.self_closing_pat.sub(self.self_closing_sub, html)
-        if self.manager is not None:
-            self.manager.load_started()
-        self.loading_url = QUrl.fromLocalFile(path)
-        html = re.sub(ur'<\s*title\s*/\s*>', u'', html, flags=re.IGNORECASE)
-        if has_svg:
-            self.setContent(QByteArray(html.encode(path.encoding)), mt, QUrl.fromLocalFile(path))
-        else:
-            self.setHtml(html, self.loading_url)
+        def callback(lu):
+            self.loading_url = lu
+            if self.manager is not None:
+                self.manager.load_started()
+
+        load_html(path, self, codec=path.encoding, mime_type=getattr(path,
+            'mime_type', None), pre_load_callback=callback)
         self.turn_off_internal_scrollbars()
 
     def initialize_scrollbar(self):
@@ -1011,8 +955,12 @@ class DocumentView(QWebView): # {{{
             finally:
                 self.is_auto_repeat_event = False
         elif key == 'Down':
+            if self.document.at_bottom:
+                self.manager.next_document()
             self.scroll_by(y=15)
         elif key == 'Up':
+            if self.document.at_top:
+                self.manager.previous_document()
             self.scroll_by(y=-15)
         elif key == 'Left':
             self.scroll_by(x=-15)

src/calibre/gui2/viewer/main.py

@@ -27,6 +27,7 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.customize.ui import available_input_formats
 from calibre.gui2.viewer.dictionary import Lookup
 from calibre import as_unicode, force_unicode, isbytestring
+from calibre.ptempfile import reset_base_dir
 
 vprefs = JSONConfig('viewer')
 
@@ -512,17 +513,18 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                     self.load_path(self.iterator.spine[spine_index])
 
     def goto_bookmark(self, bm):
-        m = bm[1].split('#')
-        if len(m) > 1:
-            spine_index, m = int(m[0]), m[1]
-            if spine_index > -1 and self.current_index == spine_index:
-                self.view.goto_bookmark(m)
+        spine_index = bm['spine']
+        if spine_index > -1 and self.current_index == spine_index:
+            if self.resize_in_progress:
+                self.view.document.page_position.set_pos(bm['pos'])
             else:
-                self.pending_bookmark = bm
-                if spine_index < 0 or spine_index >= len(self.iterator.spine):
-                    spine_index = 0
-                    self.pending_bookmark = None
-                self.load_path(self.iterator.spine[spine_index])
+                self.view.goto_bookmark(bm)
+        else:
+            self.pending_bookmark = bm
+            if spine_index < 0 or spine_index >= len(self.iterator.spine):
+                spine_index = 0
+                self.pending_bookmark = None
+            self.load_path(self.iterator.spine[spine_index])
 
     def toc_clicked(self, index):
         item = self.toc_model.itemFromIndex(index)
@@ -699,6 +701,14 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.view.load_path(path, pos=pos)
 
     def viewport_resize_started(self, event):
+        old, curr = event.size(), event.oldSize()
+        if not self.window_mode_changed and old.width() == curr.width():
+            # No relayout changes, so page position does not need to be saved
+            # This is needed as Qt generates a viewport resized event that
+            # changes only the height after a file has been loaded. This can
+            # cause the last read position bookmark to become slightly
+            # inaccurate
+            return
         if not self.resize_in_progress:
             # First resize, so save the current page position
             self.resize_in_progress = True
@@ -746,9 +756,10 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 _('Enter title for bookmark:'), text=bm)
         title = unicode(title).strip()
         if ok and title:
-            pos = self.view.bookmark()
-            bookmark = '%d#%s'%(self.current_index, pos)
-            self.iterator.add_bookmark((title, bookmark))
+            bm = self.view.bookmark()
+            bm['spine'] = self.current_index
+            bm['title'] = title
+            self.iterator.add_bookmark(bm)
             self.set_bookmarks(self.iterator.bookmarks)
 
     def set_bookmarks(self, bookmarks):
@@ -758,12 +769,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         current_page = None
         self.existing_bookmarks = []
         for bm in bookmarks:
-            if bm[0] == 'calibre_current_page_bookmark' and \
-                                self.get_remember_current_page_opt():
-                current_page = bm
+            if bm['title'] == 'calibre_current_page_bookmark':
+                if self.get_remember_current_page_opt():
+                    current_page = bm
             else:
-                self.existing_bookmarks.append(bm[0])
-                self.bookmarks_menu.addAction(bm[0], partial(self.goto_bookmark, bm))
+                self.existing_bookmarks.append(bm['title'])
+                self.bookmarks_menu.addAction(bm['title'], partial(self.goto_bookmark, bm))
         return current_page
 
     def manage_bookmarks(self):
@@ -783,9 +794,10 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             return
         if hasattr(self, 'current_index'):
             try:
-                pos = self.view.bookmark()
-                bookmark = '%d#%s'%(self.current_index, pos)
-                self.iterator.add_bookmark(('calibre_current_page_bookmark', bookmark))
+                bm = self.view.bookmark()
+                bm['spine'] = self.current_index
+                bm['title'] = 'calibre_current_page_bookmark'
+                self.iterator.add_bookmark(bm)
             except:
                 traceback.print_exc()
 
@@ -947,6 +959,7 @@ View an ebook.
 def main(args=sys.argv):
     # Ensure viewer can continue to function if GUI is closed
     os.environ.pop('CALIBRE_WORKER_TEMP_DIR', None)
+    reset_base_dir()
 
     parser = option_parser()
     opts, args = parser.parse_args(args)

src/calibre/gui2/viewer/position.py

@@ -67,10 +67,16 @@ class PagePosition(object):
 
     def restore(self):
         if self._cpos is None: return
-        if isinstance(self._cpos, (int, float)):
-            self.document.scroll_fraction = self._cpos
-        else:
-            self.scroll_to_cfi(self._cpos)
+        self.to_pos(self._cpos)
         self._cpos = None
 
+    def to_pos(self, pos):
+        if isinstance(pos, (int, float)):
+            self.document.scroll_fraction = pos
+        else:
+            self.scroll_to_cfi(pos)
+
+    def set_pos(self, pos):
+        self._cpos = pos
+
 

src/calibre/library/cli.py

@@ -233,7 +233,7 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
             if not mi.authors:
                 mi.authors = [_('Unknown')]
             for x in ('title', 'authors', 'isbn', 'tags', 'series'):
-                val = locals()[x]
+                val = locals()['o'+x]
                 if val: setattr(mi, x[1:], val)
             if oseries:
                 mi.series_index = oseries_index
@@ -356,7 +356,7 @@ def command_add(args, dbpath):
         print >>sys.stderr, _('You must specify at least one file to add')
         return 1
     do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory,
-            opts.recurse, opts.duplicates, opts.title, opts.author, opts.isbn,
+            opts.recurse, opts.duplicates, opts.title, opts.authors, opts.isbn,
             tags, opts.series, opts.series_index)
     return 0
 

src/calibre/linux.py

@@ -40,6 +40,46 @@ entry_points = {
                             ],
       }
 
+class PreserveMIMEDefaults(object):
+
+    def __init__(self):
+        self.initial_values = {}
+
+    def __enter__(self):
+        def_data_dirs = '/usr/local/share:/usr/share'
+        paths = os.environ.get('XDG_DATA_DIRS', def_data_dirs)
+        paths = paths.split(':')
+        paths.append(os.environ.get('XDG_DATA_HOME', os.path.expanduser(
+            '~/.local/share')))
+        paths = list(filter(os.path.isdir, paths))
+        if not paths:
+            # Env var had garbage in it, ignore it
+            paths = def_data_dirs.split(':')
+        paths = list(filter(os.path.isdir, paths))
+        self.paths = {os.path.join(x, 'applications/defaults.list') for x in
+                paths}
+        self.initial_values = {}
+        for x in self.paths:
+            try:
+                with open(x, 'rb') as f:
+                    self.initial_values[x] = f.read()
+            except:
+                self.initial_values[x] = None
+
+    def __exit__(self, *args):
+        for path, val in self.initial_values.iteritems():
+            if val is None:
+                try:
+                    os.remove(path)
+                except:
+                    pass
+            elif os.path.exists(path):
+                with open(path, 'r+b') as f:
+                    if f.read() != val:
+                        f.seek(0)
+                        f.truncate()
+                        f.write(val)
+
 # Uninstall script {{{
 UNINSTALL = '''\
 #!{python}
@@ -202,6 +242,10 @@ class PostInstall:
             if not os.path.exists(os.path.dirname(f)):
                 os.makedirs(os.path.dirname(f))
             self.manifest.append(f)
+            complete = 'calibre-complete'
+            if getattr(sys, 'frozen_path', None):
+                complete = os.path.join(getattr(sys, 'frozen_path'), complete)
+
             self.info('Installing bash completion to', f)
             with open(f, 'wb') as f:
                 f.write('# calibre Bash Shell Completion\n')
@@ -286,8 +330,8 @@ class PostInstall:
                 }
                 complete -o nospace  -F _ebook_device ebook-device
 
-                complete -o nospace -C calibre-complete ebook-convert
-                '''))
+                complete -o nospace -C %s ebook-convert
+                ''')%complete)
         except TypeError as err:
             if 'resolve_entities' in str(err):
                 print 'You need python-lxml >= 2.0.5 for calibre'
@@ -333,57 +377,55 @@ class PostInstall:
 
     def setup_desktop_integration(self): # {{{
         try:
-
             self.info('Setting up desktop integration...')
 
+            with TemporaryDirectory() as tdir, CurrentDir(tdir), \
+                                PreserveMIMEDefaults():
+                render_img('mimetypes/lrf.png', 'calibre-lrf.png')
+                check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png application-lrf', shell=True)
+                self.icon_resources.append(('mimetypes', 'application-lrf', '128'))
+                check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png text-lrs', shell=True)
+                self.icon_resources.append(('mimetypes', 'application-lrs',
+                '128'))
+                render_img('lt.png', 'calibre-gui.png')
+                check_call('xdg-icon-resource install --noupdate --size 128 calibre-gui.png calibre-gui', shell=True)
+                self.icon_resources.append(('apps', 'calibre-gui', '128'))
+                render_img('viewer.png', 'calibre-viewer.png')
+                check_call('xdg-icon-resource install --size 128 calibre-viewer.png calibre-viewer', shell=True)
+                self.icon_resources.append(('apps', 'calibre-viewer', '128'))
 
-            with TemporaryDirectory() as tdir:
-                with CurrentDir(tdir):
-                    render_img('mimetypes/lrf.png', 'calibre-lrf.png')
-                    check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png application-lrf', shell=True)
-                    self.icon_resources.append(('mimetypes', 'application-lrf', '128'))
-                    check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png text-lrs', shell=True)
-                    self.icon_resources.append(('mimetypes', 'application-lrs',
-                    '128'))
-                    render_img('lt.png', 'calibre-gui.png')
-                    check_call('xdg-icon-resource install --noupdate --size 128 calibre-gui.png calibre-gui', shell=True)
-                    self.icon_resources.append(('apps', 'calibre-gui', '128'))
-                    render_img('viewer.png', 'calibre-viewer.png')
-                    check_call('xdg-icon-resource install --size 128 calibre-viewer.png calibre-viewer', shell=True)
-                    self.icon_resources.append(('apps', 'calibre-viewer', '128'))
+                mimetypes = set([])
+                for x in all_input_formats():
+                    mt = guess_type('dummy.'+x)[0]
+                    if mt and 'chemical' not in mt and 'ctc-posml' not in mt:
+                        mimetypes.add(mt)
 
-                    mimetypes = set([])
-                    for x in all_input_formats():
-                        mt = guess_type('dummy.'+x)[0]
-                        if mt and 'chemical' not in mt and 'ctc-posml' not in mt:
-                            mimetypes.add(mt)
+                def write_mimetypes(f):
+                    f.write('MimeType=%s;\n'%';'.join(mimetypes))
 
-                    def write_mimetypes(f):
-                        f.write('MimeType=%s;\n'%';'.join(mimetypes))
-
-                    f = open('calibre-lrfviewer.desktop', 'wb')
-                    f.write(VIEWER)
-                    f.close()
-                    f = open('calibre-ebook-viewer.desktop', 'wb')
-                    f.write(EVIEWER)
-                    write_mimetypes(f)
-                    f.close()
-                    f = open('calibre-gui.desktop', 'wb')
-                    f.write(GUI)
-                    write_mimetypes(f)
-                    f.close()
-                    des = ('calibre-gui.desktop', 'calibre-lrfviewer.desktop',
-                            'calibre-ebook-viewer.desktop')
-                    for x in des:
-                        cmd = ['xdg-desktop-menu', 'install', '--noupdate', './'+x]
-                        check_call(' '.join(cmd), shell=True)
-                        self.menu_resources.append(x)
-                    check_call(['xdg-desktop-menu', 'forceupdate'])
-                    f = open('calibre-mimetypes', 'wb')
-                    f.write(MIME)
-                    f.close()
-                    self.mime_resources.append('calibre-mimetypes')
-                    check_call('xdg-mime install ./calibre-mimetypes', shell=True)
+                f = open('calibre-lrfviewer.desktop', 'wb')
+                f.write(VIEWER)
+                f.close()
+                f = open('calibre-ebook-viewer.desktop', 'wb')
+                f.write(EVIEWER)
+                write_mimetypes(f)
+                f.close()
+                f = open('calibre-gui.desktop', 'wb')
+                f.write(GUI)
+                write_mimetypes(f)
+                f.close()
+                des = ('calibre-gui.desktop', 'calibre-lrfviewer.desktop',
+                        'calibre-ebook-viewer.desktop')
+                for x in des:
+                    cmd = ['xdg-desktop-menu', 'install', '--noupdate', './'+x]
+                    check_call(' '.join(cmd), shell=True)
+                    self.menu_resources.append(x)
+                check_call(['xdg-desktop-menu', 'forceupdate'])
+                f = open('calibre-mimetypes', 'wb')
+                f.write(MIME)
+                f.close()
+                self.mime_resources.append('calibre-mimetypes')
+                check_call('xdg-mime install ./calibre-mimetypes', shell=True)
         except Exception:
             if self.opts.fatal_errors:
                 raise

src/calibre/ptempfile.py

@@ -74,6 +74,11 @@ def base_dir():
 
     return _base_dir
 
+def reset_base_dir():
+    global _base_dir
+    _base_dir = None
+    base_dir()
+
 def force_unicode(x):
     # Cannot use the implementation in calibre.__init__ as it causes a circular
     # dependency

src/calibre/translations/calibre.pot

@@ -5,8 +5,8 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: calibre 0.8.44\n"
-"POT-Creation-Date: 2012-03-23 08:23+IST\n"
-"PO-Revision-Date: 2012-03-23 08:23+IST\n"
+"POT-Creation-Date: 2012-03-24 16:05+IST\n"
+"PO-Revision-Date: 2012-03-24 16:05+IST\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@@ -175,7 +175,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/store/search/models.py:204
 #: /home/kovid/work/calibre/src/calibre/gui2/store/stores/google_books_plugin.py:107
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main.py:205
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:233
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:234
 #: /home/kovid/work/calibre/src/calibre/library/database.py:914
 #: /home/kovid/work/calibre/src/calibre/library/database2.py:561
 #: /home/kovid/work/calibre/src/calibre/library/database2.py:569
@@ -4279,8 +4279,10 @@ msgid "Empty output file, probably the conversion process crashed"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/copy_to_library.py:84
+#: /home/kovid/work/calibre/src/calibre/gui2/add.py:385
+#: /home/kovid/work/calibre/src/calibre/gui2/auto_add.py:214
 #, python-format
-msgid "%s by %s"
+msgid "%(title)s by %(author)s"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/copy_to_library.py:131
@@ -4712,7 +4714,7 @@ msgid "Move to next highlighted match"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/next_match.py:13
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:391
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:388
 msgid "N"
 msgstr ""
 
@@ -4729,7 +4731,7 @@ msgid "Shift+N"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/next_match.py:27
-#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:214
+#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:213
 msgid "Shift+F3"
 msgstr ""
 
@@ -5117,12 +5119,6 @@ msgstr ""
 msgid "The add books process seems to have hung. Try restarting calibre and adding the books in smaller increments, until you find the problem book."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/add.py:385
-#: /home/kovid/work/calibre/src/calibre/gui2/auto_add.py:214
-#, python-format
-msgid "%(title)s by %(author)s"
-msgstr ""
-
 #: /home/kovid/work/calibre/src/calibre/gui2/add.py:387
 #: /home/kovid/work/calibre/src/calibre/gui2/auto_add.py:216
 msgid "Duplicates found!"
@@ -5661,167 +5657,167 @@ msgstr ""
 msgid "Tab template for catalog.ui"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:70
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:77
 msgid "Bold"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:71
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:78
 msgid "Italic"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:74
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:81
 msgid "Underline"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:76
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:83
 msgid "Strikethrough"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:78
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:85
 msgid "Superscript"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:80
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:87
 msgid "Subscript"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:82
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:89
 msgid "Ordered list"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:84
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:91
 msgid "Unordered list"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:87
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:94
 msgid "Align left"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:89
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:96
 msgid "Align center"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:91
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:98
 msgid "Align right"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:93
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:100
 msgid "Align justified"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:94
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:101
 msgid "Undo"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:95
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:102
 msgid "Redo"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:96
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:103
 msgid "Remove formatting"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:97
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:104
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/edit_authors_dialog.py:174
 msgid "Copy"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:98
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:105
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/edit_authors_dialog.py:176
 msgid "Paste"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:99
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:106
 msgid "Cut"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:101
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:108
 msgid "Increase Indentation"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:103
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:110
 msgid "Decrease Indentation"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:105
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:112
 msgid "Select all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:110
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:120
 msgid "Foreground color"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:115
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:125
 msgid "Background color"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:119
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:129
 msgid "Style text block"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:121
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:131
 msgid "Style the selected text block"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:126
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:136
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/behavior.py:34
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/behavior.py:36
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/behavior_ui.py:158
 msgid "Normal"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:127
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:128
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:129
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:130
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:131
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:132
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:137
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:138
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:139
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:140
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:141
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:142
 msgid "Heading"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:133
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:143
 msgid "Pre-formatted"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:134
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:144
 msgid "Blockquote"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:135
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:145
 msgid "Address"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:142
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:152
 msgid "Insert link"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:144
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:154
 #: /home/kovid/work/calibre/src/calibre/gui2/shortcuts_ui.py:79
 #: /home/kovid/work/calibre/src/calibre/gui2/shortcuts_ui.py:84
 msgid "Clear"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:162
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:172
 msgid "Choose foreground color"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:168
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:178
 msgid "Choose background color"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:173
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:183
 msgid "Create link"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:174
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:184
 msgid "Enter URL"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:528
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:538
 msgid "Normal view"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:529
+#: /home/kovid/work/calibre/src/calibre/gui2/comments_editor.py:539
 msgid "HTML Source"
 msgstr ""
 
@@ -7081,9 +7077,9 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/metadata_bulk.py:349
 #: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:83
 #: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:103
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:225
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:274
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:278
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:222
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:271
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:275
 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/basic_widgets.py:1413
 msgid "Undefined"
 msgstr ""
@@ -7375,14 +7371,14 @@ msgid "You have enabled the <b>{0}</b> formats for your {1}. The {1} may not sup
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/device_drivers/configwidget.py:150
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:440
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:437
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/plugboard.py:279
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/save_template.py:70
 msgid "Invalid template"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/device_drivers/configwidget.py:151
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:441
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:438
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/plugboard.py:280
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/save_template.py:71
 #, python-format
@@ -8118,7 +8114,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:186
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:237
 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:869
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:970
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:975
 msgid "View log"
 msgstr ""
 
@@ -9648,7 +9644,7 @@ msgid "Open Template Editor"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/template_line_editor.py:41
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:427
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:424
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/save_template.py:48
 msgid "Edit template"
 msgstr ""
@@ -10363,7 +10359,7 @@ msgstr ""
 msgid "stars"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:391
+#: /home/kovid/work/calibre/src/calibre/gui2/library/delegates.py:388
 msgid "Y"
 msgstr ""
 
@@ -11173,7 +11169,7 @@ msgstr ""
 msgid "Downloading metadata..."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:954
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:959
 msgid "Downloading cover..."
 msgstr ""
 
@@ -14745,18 +14741,14 @@ msgid "Toggle full screen"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:210
-msgid "Toggle full screen (F11)"
-msgstr ""
-
-#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:211
 msgid "Print"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:212
+#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:211
 msgid "Find previous"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:213
+#: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:212
 msgid "Find previous occurrence"
 msgstr ""
 
@@ -15519,7 +15511,7 @@ msgid "Filter the results by the search query. For the format of the search quer
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/library/cli.py:159
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1063
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1086
 msgid "The maximum width of a single line in the output. Defaults to detecting screen size."
 msgstr ""
 
@@ -15539,11 +15531,11 @@ msgstr ""
 msgid "Invalid sort field. Available fields:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:264
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:270
 msgid "The following books were not added as they already exist in the database (see --duplicates option):"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:289
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:295
 msgid ""
 "%prog add [options] file1 file2 file3 ...\n"
 "\n"
@@ -15551,39 +15543,51 @@ msgid ""
 "the directory related options below.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:297
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:303
 msgid "Assume that each directory has only a single logical book and that all files in it are different e-book formats of that book"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:299
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:305
 msgid "Process directories recursively"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:301
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:307
 msgid "Add books to database even if they already exist. Comparison is done based on book titles."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:303
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:309
 msgid "Add an empty book (a book with no formats)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:305
-msgid "Set the title of the added empty book"
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:311
+msgid "Set the title of the added book(s)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:307
-msgid "Set the authors of the added empty book"
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:313
+msgid "Set the authors of the added book(s)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:309
-msgid "Set the ISBN of the added empty book"
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:315
+msgid "Set the ISBN of the added book(s)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:335
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:317
+msgid "Set the tags of the added book(s)"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:319
+msgid "Set the series of the added book(s)"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:321
+msgid "Set the series number of the added book(s)"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:356
 msgid "You must specify at least one file to add"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:353
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:376
 msgid ""
 "%prog remove ids\n"
 "\n"
@@ -15591,26 +15595,26 @@ msgid ""
 "included).\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:368
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:391
 msgid "You must specify at least one book to remove"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:389
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:412
 msgid ""
 "%prog add_format [options] id ebook_file\n"
 "\n"
 "Add the ebook in ebook_file to the available formats for the logical book identified by id. You can get id by using the list command. If the format already exists, it is replaced.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:403
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:426
 msgid "You must specify an id and an ebook file"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:408
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:431
 msgid "ebook file must have an extension"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:418
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:441
 msgid ""
 "\n"
 "%prog remove_format [options] id fmt\n"
@@ -15618,11 +15622,11 @@ msgid ""
 "Remove the format fmt from the logical book identified by id. You can get id by using the list command. fmt should be a file extension like LRF or TXT or EPUB. If the logical book does not have fmt available, do nothing.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:434
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:457
 msgid "You must specify an id and a format"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:453
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:476
 msgid ""
 "\n"
 "%prog show_metadata [options] id\n"
@@ -15631,15 +15635,15 @@ msgid ""
 "id is an id number from the list command.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:460
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:483
 msgid "Print metadata in OPF form (XML)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:469
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:492
 msgid "You must specify an id"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:485
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:508
 msgid ""
 "\n"
 "%prog set_metadata [options] id /path/to/metadata.opf\n"
@@ -15650,11 +15654,11 @@ msgid ""
 "show_metadata command.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:500
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:523
 msgid "You must specify an id and a metadata file"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:520
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:543
 msgid ""
 "%prog export [options] ids\n"
 "\n"
@@ -15663,28 +15667,28 @@ msgid ""
 "an opf file). You can get id numbers from the list command.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:528
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:551
 msgid "Export all books in database, ignoring the list of ids."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:530
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:553
 msgid "Export books to the specified directory. Default is"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:532
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:555
 msgid "Export all books into a single directory"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:539
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:562
 msgid "Specifying this switch will turn this behavior off."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:562
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:585
 #, python-format
 msgid "You must specify some ids or the %s option"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:575
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:598
 msgid ""
 "%prog add_custom_column [options] label name datatype\n"
 "\n"
@@ -15693,19 +15697,19 @@ msgid ""
 "datatype is one of: {0}\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:584
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:607
 msgid "This column stores tag like data (i.e. multiple comma separated values). Only applies if datatype is text."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:588
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:611
 msgid "A dictionary of options to customize how the data in this column will be interpreted. This is a JSON  string. For enumeration columns, use --display='{\"enum_values\":[\"val1\", \"val2\"]}'"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:602
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:625
 msgid "You must specify label, name and datatype"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:664
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:687
 msgid ""
 "\n"
 "    %prog catalog /path/to/destination.(CSV|EPUB|MOBI|XML ...) [options]\n"
@@ -15715,29 +15719,29 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:677
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:700
 msgid ""
 "Comma-separated list of database IDs to catalog.\n"
 "If declared, --search is ignored.\n"
 "Default: all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:681
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:704
 msgid ""
 "Filter the results by the search query. For the format of the search query, please see the search-related documentation in the User Manual.\n"
 "Default: no filtering"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:687
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:710
 #: /home/kovid/work/calibre/src/calibre/web/fetch/simple.py:528
 msgid "Show detailed output information. Useful for debugging"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:700
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:723
 msgid "Error: You must specify a catalog output file"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:747
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:770
 msgid ""
 "\n"
 "    %prog set_custom [options] column id value\n"
@@ -15749,15 +15753,15 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:757
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:780
 msgid "If the column stores multiple values, append the specified values to the existing ones, instead of replacing them."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:768
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:791
 msgid "Error: You must specify a field name, id and value"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:788
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:811
 msgid ""
 "\n"
 "    %prog custom_columns [options]\n"
@@ -15766,20 +15770,20 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:794
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:817
 msgid "Show details for each column."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:806
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:829
 #, python-format
 msgid "You will lose all data in the column: %r. Are you sure (y/n)? "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:808
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:831
 msgid "y"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:815
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:838
 msgid ""
 "\n"
 "    %prog remove_custom_column [options] label\n"
@@ -15789,15 +15793,15 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:822
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:845
 msgid "Do not ask for confirmation"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:832
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:855
 msgid "Error: You must specify a column label"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:843
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:866
 msgid ""
 "\n"
 "    %prog saved_searches [options] list\n"
@@ -15810,74 +15814,74 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:860
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:883
 msgid "Error: You must specify an action (add|remove|list)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:868
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:891
 msgid "Name:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:869
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:892
 msgid "Search string:"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:875
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:898
 msgid "Error: You must specify a name and a search string"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:878
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:901
 msgid "added"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:883
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:906
 msgid "Error: You must specify a name"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:886
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:909
 msgid "removed"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:890
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:913
 #, python-format
 msgid "Error: Action %s not recognized, must be one of: (add|remove|list)"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:898
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:921
 msgid ""
 "%prog check_library [options]\n"
 "\n"
 "Perform some checks on the filesystem representing a library. Reports are {0}\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:905
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1055
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:928
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1078
 msgid "Output in CSV"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:908
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:931
 msgid ""
 "Comma-separated list of reports.\n"
 "Default: all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:912
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:935
 msgid ""
 "Comma-separated list of extensions to ignore.\n"
 "Default: all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:916
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:939
 msgid ""
 "Comma-separated list of names to ignore.\n"
 "Default: all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:946
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:969
 msgid "Unknown report check"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:980
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1003
 msgid ""
 "%prog restore_database [options]\n"
 "\n"
@@ -15892,16 +15896,16 @@ msgid ""
 "    "
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:994
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1017
 msgid "Really do the recovery. The command will not run unless this option is specified."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1007
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1030
 #, python-format
 msgid "You must provide the %s option to do a recovery"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1044
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1067
 msgid ""
 "%prog list_categories [options]\n"
 "\n"
@@ -15909,29 +15913,29 @@ msgid ""
 "information is the equivalent of what is shown in the tags pane.\n"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1052
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1075
 msgid "Output only the number of items in a category instead of the counts per item within the category"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1057
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1080
 msgid "The character to put around the category value in CSV mode. Default is quotes (\")."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1060
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1083
 msgid ""
 "Comma-separated list of category lookup names.\n"
 "Default: all"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1066
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1089
 msgid "The string used to separate fields in CSV mode. Default is a comma."
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1104
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1127
 msgid "CATEGORY ITEMS"
 msgstr ""
 
-#: /home/kovid/work/calibre/src/calibre/library/cli.py:1177
+#: /home/kovid/work/calibre/src/calibre/library/cli.py:1200
 #, python-format
 msgid ""
 "%%prog command [options] [arguments]\n"
@@ -16702,7 +16706,7 @@ msgid "ondevice() -- return Yes if ondevice is set, otherwise return the empty s
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:844
-msgid "booksize() -- return the series sort value"
+msgid "series_sort() -- return the series sort value"
 msgstr ""
 
 #: /home/kovid/work/calibre/src/calibre/utils/formatter_functions.py:855

src/calibre/web/feeds/recipes/collection.py

@@ -437,6 +437,14 @@ class SchedulerConfig(object):
                 if x.get('id', False) == urn:
                     return x.get('username', ''), x.get('password', '')
 
+    def clear_account_info(self, urn):
+        with self.lock:
+            for x in self.iter_accounts():
+                 if x.get('id', False) == urn:
+                     x.getparent().remove(x)
+                     self.write_scheduler_file()
+                     break
+
     def get_customize_info(self, urn):
         keep_issues = 0
         add_title_tag = True

src/calibre/web/feeds/recipes/model.py

@@ -354,6 +354,9 @@ class RecipeModel(QAbstractItemModel, SearchQueryParser):
     def set_account_info(self, urn, un, pw):
         self.scheduler_config.set_account_info(urn, un, pw)
 
+    def clear_account_info(self, urn):
+        self.scheduler_config.clear_account_info(urn)
+
     def get_account_info(self, urn):
         return self.scheduler_config.get_account_info(urn)