merge from trunk

2025-07-09 03:04:10 -04:00 · 2012-04-28 22:07:30 +10:00 · 2012-04-28 22:07:30 +10:00 · f3f689570b
commit f3f689570b
parent f0708f779e ab3ec3640d
118 changed files with 34299 additions and 32246 deletions
--- a/Changelog.yaml
+++ b/Changelog.yaml
@ -19,6 +19,51 @@
 #   new recipes:
 #     - title: 

+- version: 0.8.47
+  date: 2012-04-13
+
+  new features:
+    - title: "Conversion pipeline: Add support for all the named entities in the HTML 5 spec."
+      tickets: [976056]
+
+    - title: "Support for viewing and converting the Haodoo PDB ebook format"
+      tickets: [976478]
+
+    - title: "Device driver for Laser EB720"
+ 
+  bug fixes:
+    - title: "Fix regression in automatic adding in 0.8.46 that broke automatic adding if adding of duplicates is enabled and auto convert is also enabled"
+      tickets: [976336]
+
+    - title: 'Fix "Tags" field in advanced search does not obey regex setting'
+      tickets: [980221]
+
+    - title: "EPUB Input: Automatically extract cover image from simple HTML title page that consists of only a single <img> tag, instead of rendering the page"
+
+    - title: "Prevent errors when both author and author_sort are used in a template for reading metadata from filenames for files on a device"
+
+    - title: "Amazon metadata download: Handle books whose titles start with a bracket."
+      tickets: [976365]
+
+    - title: "Get Books: Fix downloading of purchased books from Baen"
+      tickets: [975929]
+
+
+  improved recipes:
+    - Forbes
+    - Caros Amigos
+    - Trouw
+    - Sun UK
+    - Metro
+    - Daily Mirror
+
+  new recipes:
+    - title: "Melbourne Herald Sun"
+      author: Ray Hartley
+
+    - title: "Editoriali and Zerocalcare"
+      author: faber1971
+
 - version: 0.8.46
  date: 2012-04-06

--- a/recipes/daily_mirror.recipe
+++ b/recipes/daily_mirror.recipe
@ -1,20 +1,21 @@
+
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
+import mechanize
 class AdvancedUserRecipe1306061239(BasicNewsRecipe):
    title          = u'The Daily Mirror'
    description = 'News as provide by The Daily Mirror -UK'

    __author__ = 'Dave Asbury'
-    # last updated 11/2/12
+    # last updated 7/4/12
    language = 'en_GB'
-
-    cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
+    #cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'

    masthead_url = 'http://www.nmauk.co.uk/nma/images/daily_mirror.gif'


    oldest_article = 1
-    max_articles_per_feed = 5
+    max_articles_per_feed = 10
    remove_empty_feeds = True
    remove_javascript     = True
    no_stylesheets = True
@ -75,3 +76,28 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe):
                    img { display:block}
                	 '''

+    def get_cover_url(self):
+        soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html')
+        # look for the block containing the mirror button and url
+        cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'})
+        cov2 = str(cov)
+        cov2='http://www.politicshome.com'+cov2[9:-142]
+        #cov2 now contains url of the page containing pic
+        soup = self.index_to_soup(cov2)
+        cov = soup.find(attrs={'id' : 'large'})
+        cov2 = str(cov)
+        cov2=cov2[27:-18]
+        #cov2 now is pic url, now  go back to original function
+        br = mechanize.Browser()
+        br.set_handle_redirect(False)
+        try:
+                br.open_novisit(cov2)
+                cover_url = cov2
+        except:
+                cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg'
+
+        #cover_url = cov2
+        #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
+        return cover_url
+
+
--- a/recipes/forbes.recipe
+++ b/recipes/forbes.recipe
@ -1,39 +1,49 @@
-from calibre.ebooks.BeautifulSoup import BeautifulSoup
+import re
 from calibre.web.feeds.news import BasicNewsRecipe

 class Forbes(BasicNewsRecipe):
    title          = u'Forbes'
    description    = 'Business and Financial News'
-    __author__     = 'Darko Miletic'
+    __author__     = 'Kovid Goyal'
    oldest_article = 30
-    max_articles_per_feed = 100
+    max_articles_per_feed = 20
    language = 'en'
+    encoding = 'utf-8'
+    recursions = 1

    no_stylesheets = True
-    html2lrf_options = ['--base-font-size', '10']     

    cover_url  = u'http://www.forbes.com/media/current_covers/forbes_120_160.gif'
-    
-    feeds          = [(u'Latest', u'http://www.forbes.com/news/index.xml'), 
-        (u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'), 
-        (u'Most Emailed', u'http://www.forbes.com/feeds/mostemailed.xml'), 
-        (u'Faces', u'http://www.forbes.com/facesscan/index.xml'), 
-        (u'Technology', u'http://www.forbes.com/technology/index.xml'), 
-        (u'Personal Tech', u'http://www.forbes.com/personaltech/index.xml'), 
-        (u'Wireless', u'http://www.forbes.com/wireless/index.xml'),
-        (u'Business', u'http://www.forbes.com/business/index.xml'), 
-        (u'Sports Money', u'http://www.forbes.com/sportsmoney/index.xml'), 
-        (u'Sports', u'http://www.forbes.com/forbeslife/sports/index.xml'),
-        (u'Vehicles', u'http://www.forbes.com/forbeslife/vehicles/index.xml'),
-        (u'Leadership', u'http://www.forbes.com/leadership/index.xml'), 
-        (u'Careers', u'http://www.forbes.com/leadership/careers/index.xml'),
-        (u'Compensation', u'http://www.forbes.com/leadership/compensation/index.xml'),
-        (u'Managing', u'http://www.forbes.com/leadership/managing/index.xml')]

-    def print_version(self, url):
-        raw = self.browser.open(url).read()
-        soup = BeautifulSoup(raw.decode('latin1', 'replace'))
-        print_link = soup.find('a', {'onclick':"s_linkTrackVars='prop18';s_linkType='o';s_linkName='Print';if(typeof(globalPageName)!='undefined')s_prop18=globalPageName;s_lnk=s_co(this);s_gs(s_account);"})
-        if print_link is None:
-            return ''
-        return 'http://www.forbes.com' + print_link['href']
+    feeds          = [(u'Latest', u'http://www.forbes.com/news/index.xml'),
+        (u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'),
+        (u'Technology', u'http://www.forbes.com/technology/index.xml'),
+        (u'Business', u'http://www.forbes.com/business/index.xml'),
+        (u'Sports Money', u'http://www.forbes.com/sportsmoney/index.xml'),
+        (u'Leadership', u'http://www.forbes.com/leadership/index.xml'),]
+
+    keep_only_tags = \
+            {'class':lambda x: x and (set(x.split()) & {'body', 'pagination',
+                'articleHead', 'article_head'})}
+    remove_tags_before = {'name':'h1'}
+    remove_tags = [
+            {'class':['comment_bug', 'engagement_block',
+                        'video_promo_block', 'article_actions']},
+            {'id':'comments'}
+        ]
+
+    def is_link_wanted(self, url, tag):
+        ans = re.match(r'http://.*/[2-9]/', url) is not None
+        if ans:
+            self.log('Following multipage link: %s'%url)
+        return ans
+
+    def postprocess_html(self, soup, first_fetch):
+        for pag in soup.findAll(True, 'pagination'):
+            pag.extract()
+        if not first_fetch:
+            h1 = soup.find('h1')
+            if h1 is not None:
+                h1.extract()
+        return soup
+
--- a/recipes/jakarta_globe.recipe
+++ b/recipes/jakarta_globe.recipe
@ -0,0 +1,34 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class JakartaGlobe(BasicNewsRecipe):
+    title          = u'Jakarta Globe'
+    oldest_article = 3
+    max_articles_per_feed = 100
+
+    feeds          = [
+	(u'News', u'http://www.thejakartaglobe.com/pages/getrss/getrss-news.php'),
+	(u'Business', u'http://www.thejakartaglobe.com/pages/getrss/getrss-business.php'),
+	(u'Technology', u'http://www.thejakartaglobe.com/pages/getrss/getrss-tech.php'),
+	(u'My Jakarta', u'http://www.thejakartaglobe.com/pages/getrss/getrss-myjakarta.php'),
+	(u'International', u'http://www.thejakartaglobe.com/pages/getrss/getrss-international.php'),
+	(u'Life and Times', u'http://www.thejakartaglobe.com/pages/getrss/getrss-lifeandtimes.php'),
+	]
+    __author__ = 'rty'
+    pubisher  = 'JakartaGlobe.com'
+    description           = 'JakartaGlobe, Indonesia, Newspaper'
+    category              = 'News, Indonesia'
+
+
+    remove_javascript = True
+    use_embedded_content   = False
+    no_stylesheets = True
+    language = 'en_ID'
+    encoding               = 'utf-8'
+    conversion_options = {'linearize_tables':True}
+    masthead_url = 'http://www.thejakartaglobe.com/pages/2010/images/jak-globe-logo.jpg'
+    keep_only_tags = [
+							dict(name='div', attrs={'class':'story'}),
+							dict(name='span', attrs={'class':'headline'}),
+							dict(name='div', attrs={'class':'story'}),
+							dict(name='p', attrs={'id':'bodytext'})
+                               ]
--- a/recipes/metro_uk.recipe
+++ b/recipes/metro_uk.recipe
@ -1,52 +1,30 @@
-import re
 from calibre.web.feeds.news import BasicNewsRecipe
 class AdvancedUserRecipe1306097511(BasicNewsRecipe):
    title          = u'Metro UK'
    description = 'News as provide by The Metro -UK'
-
+    #timefmt = ''
    __author__ = 'Dave Asbury'
-    #last update 3/12/11
    cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg'
-    no_stylesheets = True
+    #no_stylesheets = True
    oldest_article = 1
-    max_articles_per_feed = 20
+    max_articles_per_feed = 10
    remove_empty_feeds = True
    remove_javascript     = True
+    auto_cleanup = True

-    #preprocess_regexps = [(re.compile(r'Tweet'), lambda  a : '')]
-    preprocess_regexps = [
-    (re.compile(r'<span class="img-cap legend">', re.IGNORECASE | re.DOTALL), lambda match: '<p></p><span class="img-cap legend"> ')]
-    preprocess_regexps = [
-    (re.compile(r'tweet', re.IGNORECASE | re.DOTALL), lambda match: '')]

    language = 'en_GB'
-
-
    masthead_url        = 'http://e-edition.metro.co.uk/images/metro_logo.gif'
-
-
    keep_only_tags = [
-    dict(name='h1'),dict(name='h2', attrs={'class':'h2'}),
-                    dict(attrs={'class':['img-cnt figure']}),
-        dict(attrs={'class':['art-img']}),
-                    dict(name='div', attrs={'class':'art-lft'}),
-                    dict(name='p')
+
    ]
    remove_tags    = [
-                             dict(name = 'div',attrs={'id' : ['comments-news','formSubmission']}),
-                             dict(name='div', attrs={'class':[ 'news m12 clrd clr-b p5t shareBtm', 'commentForm', 'metroCommentInnerWrap',
-                             'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r','username','clrd' ]}),
-              dict(attrs={'class':['username', 'metroCommentFormWrap','commentText','commentsNav','avatar','submDateAndTime','addYourComment','displayName']})
-                              ,dict(name='div', attrs={'class' : 'clrd art-fd fd-gr1-b'})
+
                               ]
+
+
    feeds          = [
        (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')]
-
    extra_css  = '''
-                    body {font: sans-serif medium;}'
-    h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;}
-                h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; }
-                    span{ font-size:9.5px; font-weight:bold;font-style:italic}
-                    p { text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
-
-     '''
+	body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
+                	 '''
--- a/recipes/the_sun.recipe
+++ b/recipes/the_sun.recipe
@ -1,9 +1,8 @@
-import re
+import re, mechanize
 from calibre.web.feeds.recipes import BasicNewsRecipe
 class AdvancedUserRecipe1325006965(BasicNewsRecipe):

    title          = u'The Sun UK'
-    cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'

    description = 'A Recipe for The Sun tabloid UK'
    __author__ = 'Dave Asbury'
@ -24,37 +23,69 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe):
    no_stylesheets = True

    extra_css  = '''
-	body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
-                	 '''
+    body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;}
+                     '''

    preprocess_regexps = [
-    	(re.compile(r'<div class="foot-copyright".*?</div>', re.IGNORECASE | re.DOTALL), lambda match: '')]
+        (re.compile(r'<div class="foot-copyright".*?</div>', re.IGNORECASE | re.DOTALL), lambda match: '')]



    keep_only_tags = [
                               dict(name='h1'),dict(name='h2',attrs={'class' : 'medium centered'}),
-	           dict(name='div',attrs={'class' : 'text-center'}),
-	           dict(name='div',attrs={'id' : 'bodyText'})
-	           # dict(name='p')
-	           ]
+               dict(name='div',attrs={'class' : 'text-center'}),
+               dict(name='div',attrs={'id' : 'bodyText'})
+               # dict(name='p')
+               ]
    remove_tags=[
-	       #dict(name='head'),
-	       dict(attrs={'class' : ['mystery-meat-link','ltbx-container','ltbx-var ltbx-hbxpn','ltbx-var ltbx-nav-loop','ltbx-var ltbx-url']}),
+           #dict(name='head'),
+           dict(attrs={'class' : ['mystery-meat-link','ltbx-container','ltbx-var ltbx-hbxpn','ltbx-var ltbx-nav-loop','ltbx-var ltbx-url']}),
                           dict(name='div',attrs={'class' : 'cf'}),
-	       dict(attrs={'title' : 'download flash'}),
+           dict(attrs={'title' : 'download flash'}),
                           dict(attrs={'style' : 'padding: 5px'})

-	       ]
+           ]


    feeds          = [
-	#(u'News', u'http://www.thesun.co.uk/sol/homepage/news/rss'),
-	(u'News','http://feed43.com/2517447382644748.xml'),
-	(u'Sport', u'http://feed43.com/4283846255668687.xml'),
-	(u'Bizarre', u'http://feed43.com/0233840304242011.xml'),
-	(u'Film',u'http://feed43.com/1307545221226200.xml'),
-        	(u'Music',u'http://feed43.com/1701513435064132.xml'),
-	(u'Sun Woman',u'http://feed43.com/0022626854226453.xml'),
-]
+            (u'News','http://feed43.com/2517447382644748.xml'),
+            (u'Sport', u'http://feed43.com/4283846255668687.xml'),
+            (u'Bizarre', u'http://feed43.com/0233840304242011.xml'),
+            (u'Film',u'http://feed43.com/1307545221226200.xml'),
+            (u'Music',u'http://feed43.com/1701513435064132.xml'),
+            (u'Sun Woman',u'http://feed43.com/0022626854226453.xml'),
+    ]
+
+    def get_cover_url(self):
+        soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html')
+        # look for the block containing the sun button and url
+        cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_84.gif);'})
+
+
+
+        #cov = soup.find(attrs={'id' : 'large'})
+        cov2 = str(cov)
+
+        cov2='http://www.politicshome.com'+cov2[9:-133]
+        #cov2 now contains url of the page containing pic
+
+        #cov2 now contains url of the page containing pic
+        soup = self.index_to_soup(cov2)
+        cov = soup.find(attrs={'id' : 'large'})
+        cov2 = str(cov)
+        cov2=cov2[27:-18]
+        #cov2 now is pic url, now  go back to original function
+
+        br = mechanize.Browser()
+        br.set_handle_redirect(False)
+        try:
+            br.open_novisit(cov2)
+            cover_url = cov2
+        except:
+            cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
+
+        #cover_url = cov2
+        #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png'
+        return cover_url
+

--- a/setup/hosting.py
+++ b/setup/hosting.py
@ -26,7 +26,7 @@ def login_to_google(username, password):
    br.form['Email'] = username
    br.form['Passwd'] = password
    raw = br.submit().read()
-    if re.search(br'<title>.*?Account Settings</title>', raw) is None:
+    if re.search(br'(?i)<title>.*?Account Settings</title>', raw) is None:
        x = re.search(br'(?is)<title>.*?</title>', raw)
        if x is not None:
            print ('Title of post login page: %s'%x.group())
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 46)
+numeric_version = (0, 8, 47)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"

--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@ -259,7 +259,7 @@ class LRXMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):

    name        = 'Read MOBI metadata'
-    file_types  = set(['mobi', 'prc', 'azw', 'azw4', 'pobi'])
+    file_types  = set(['mobi', 'prc', 'azw', 'azw3', 'azw4', 'pobi'])
    description = _('Read metadata from %s files')%'MOBI'

    def get_metadata(self, stream, ftype):
--- a/src/calibre/devices/android/driver.py
+++ b/src/calibre/devices/android/driver.py
@ -40,6 +40,7 @@ class ANDROID(USBMS):
                       0xcac  : [0x100, 0x0227, 0x0226, 0x222],
                       0xccf  : [0x100, 0x0227, 0x0226, 0x222],
                       0x2910 : [0x222],
+                       0xff9  : [0x9999],
            },

            # Eken
--- a/src/calibre/devices/usbms/driver.py
+++ b/src/calibre/devices/usbms/driver.py
@ -402,19 +402,23 @@ class USBMS(CLI, Device):
    def build_template_regexp(cls):
        def replfunc(match, seen=None):
            v = match.group(1)
-            if v in ['title', 'series', 'series_index', 'isbn']:
+            if v in ['authors', 'author_sort']:
+                v = 'author'
+            if v in ('title', 'series', 'series_index', 'isbn', 'author'):
                if v not in seen:
-                    seen |= set([v])
+                    seen.add(v)
                    return '(?P<' + v + '>.+?)'
-            elif v in ['authors', 'author_sort']:
-                if v not in seen:
-                    seen |= set([v])
-                    return '(?P<author>.+?)'
            return '(.+?)'
        s = set()
        f = functools.partial(replfunc, seen=s)
-        template = cls.save_template().rpartition('/')[2]
-        return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+        template = None
+        try:
+            template = cls.save_template().rpartition('/')[2]
+            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
+        except:
+            prints(u'Failed to parse template: %r'%template)
+            template = u'{title} - {authors}'
+            return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')

    @classmethod
    def path_to_unicode(cls, path):
--- a/src/calibre/ebooks/init.py
+++ b/src/calibre/ebooks/init.py
@ -31,7 +31,7 @@ BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'ht
                   'epub', 'fb2', 'djv', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                   'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb',
                   'xps', 'oxps', 'azw4', 'book', 'zbf', 'pobi', 'docx', 'md',
-                   'textile', 'markdown', 'ibook', 'iba']
+                   'textile', 'markdown', 'ibook', 'iba', 'azw3']

 class HTMLRenderer(object):

@ -93,6 +93,20 @@ def extract_calibre_cover(raw, base, log):
        if os.path.exists(img):
            return open(img, 'rb').read()

+    # Look for a simple cover, i.e. a body with no text and only one <img> tag
+    if matches is None:
+        body = soup.find('body')
+        if body is not None:
+            text = u''.join(map(unicode, body.findAll(text=True)))
+            if text.strip():
+                # Body has text, abort
+                return
+            images = body.findAll('img', src=True)
+            if 0 < len(images) < 2:
+                img = os.path.join(base, *images[0]['src'].split('/'))
+                if os.path.exists(img):
+                    return open(img, 'rb').read()
+
 def render_html_svg_workaround(path_to_html, log, width=590, height=750):
    from calibre.ebooks.oeb.base import SVG_NS
    raw = open(path_to_html, 'rb').read()
@ -108,6 +122,7 @@ def render_html_svg_workaround(path_to_html, log, width=590, height=750):
            data = extract_calibre_cover(raw, os.path.dirname(path_to_html), log)
        except:
            pass
+
    if data is None:
        renderer = render_html(path_to_html, width, height)
        data = getattr(renderer, 'data', None)
--- a/src/calibre/ebooks/conversion/cli.py
+++ b/src/calibre/ebooks/conversion/cli.py
@ -212,6 +212,7 @@ def add_pipeline_options(parser, plumber):
            if rec.level < rec.HIGH:
                option_recommendation_to_cli_option(add_option, rec)

+
 def option_parser():
    parser = OptionParser(usage=USAGE)
    parser.add_option('--list-recipes', default=False, action='store_true',
@ -272,6 +273,34 @@ def abspath(x):
        return x
    return os.path.abspath(os.path.expanduser(x))

+def read_sr_patterns(path, log=None):
+    import json, re, codecs
+    pats = []
+    with codecs.open(path, 'r', 'utf-8') as f:
+        pat = None
+        for line in f.readlines():
+            if line.endswith(u'\n'):
+                line = line[:-1]
+
+            if pat is None:
+                if not line.strip():
+                    continue
+                try:
+                    re.compile(line)
+                except:
+                    msg = u'Invalid regular expression: %r from file: %r'%(
+                            line, path)
+                    if log is not None:
+                        log.error(msg)
+                        raise SystemExit(1)
+                    else:
+                        raise ValueError(msg)
+                pat = line
+            else:
+                pats.append((pat, line))
+                pat = None
+    return json.dumps(pats)
+
 def main(args=sys.argv):
    log = Log()
    parser, plumber = create_option_parser(args, log)
@ -279,6 +308,9 @@ def main(args=sys.argv):
    for x in ('read_metadata_from_opf', 'cover'):
        if getattr(opts, x, None) is not None:
            setattr(opts, x, abspath(getattr(opts, x)))
+    if opts.search_replace:
+        opts.search_replace = read_sr_patterns(opts.search_replace, log)
+
    recommendations = [(n.dest, getattr(opts, n.dest),
                        OptionRecommendation.HIGH) \
                                        for n in parser.options_iter()
--- a/src/calibre/ebooks/conversion/plugins/mobi_input.py
+++ b/src/calibre/ebooks/conversion/plugins/mobi_input.py
@ -28,7 +28,7 @@ class MOBIInput(InputFormatPlugin):
    name        = 'MOBI Input'
    author      = 'Kovid Goyal'
    description = 'Convert MOBI files (.mobi, .prc, .azw) to HTML'
-    file_types  = set(['mobi', 'prc', 'azw'])
+    file_types  = set(['mobi', 'prc', 'azw', 'azw3'])

    def convert(self, stream, options, file_ext, log,
                accelerators):
--- a/src/calibre/ebooks/conversion/plugins/mobi_output.py
+++ b/src/calibre/ebooks/conversion/plugins/mobi_output.py
@ -153,11 +153,22 @@ class MOBIOutput(OutputFormatPlugin):

    def convert(self, oeb, output_path, input_plugin, opts, log):
        self.log, self.opts, self.oeb = log, opts, oeb
+
+        kf8 = self.create_kf8()
+        self.write_mobi(input_plugin, output_path, kf8)
+
+    def create_kf8(self):
+        from calibre.ebooks.mobi.writer8.main import KF8Writer
+        return KF8Writer(self.oeb, self.opts)
+
+    def write_mobi(self, input_plugin, output_path, kf8):
        from calibre.ebooks.mobi.mobiml import MobiMLizer
        from calibre.ebooks.oeb.transforms.manglecase import CaseMangler
        from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable
        from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder
        from calibre.customize.ui import plugin_for_input_format
+
+        opts, oeb = self.opts, self.oeb
        if not opts.no_inline_toc:
            tocadder = HTMLTOCAdder(title=opts.toc_title, position='start' if
                    opts.mobi_toc_at_start else 'end')
--- a/src/calibre/ebooks/conversion/plumber.py
+++ b/src/calibre/ebooks/conversion/plumber.py
@ -599,6 +599,7 @@ OptionRecommendation(name='renumber_headings',
    help=_('Looks for occurrences of sequential <h1> or <h2> tags. '
           'The tags are renumbered to prevent splitting in the middle '
           'of chapter headings.')),
+
 OptionRecommendation(name='sr1_search',
    recommended_value='', level=OptionRecommendation.LOW,
    help=_('Search pattern (regular expression) to be replaced with '
@ -627,18 +628,12 @@ OptionRecommendation(name='sr3_replace',
    help=_('Replacement to replace the text found with sr3-search.')),

 OptionRecommendation(name='search_replace',
-    recommended_value='[]', level=OptionRecommendation.LOW,
-    help=_('Modify the document text and structure using user defined patterns.'
-           'This option accepts parameters in two forms:\n'
-           '1.file:<path to search/replace definitions file>\n'
-           'The file should contain alternating lines or search/replace strings:\n'
-           '      <search>\n'
-           '      <replace>\n'
-           '      <search>\n'
-           '      <replace>\n'
-           'Files saved through the user interface dialog can be used with this option.\n'
-           '2.json:<json encoded list containg [search, replace] touples:\n'
-           '      json:[["search","replace"],["search","replace"]]\n')),
+    recommended_value=None, level=OptionRecommendation.LOW, help=_(
+        'Path to a file containing search and replace regular expressions. '
+        'The file must contain alternating lines of regular expression '
+        'followed by replacement pattern (which can be an empty line). '
+        'The regular expression must be in the python regex syntax and '
+        'the file must be UTF-8 encoded.')),
 ]
        # }}}

--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'

-import functools, re, search_replace_option
+import functools, re, json

 from calibre import entity_to_unicode, as_unicode

@ -517,28 +517,30 @@ class HTMLPreProcessor(object):

        # Function for processing search and replace
        def do_search_replace(search_pattern, replace_txt):
-            if search_pattern:
-                try:
-                    search_re = re.compile(search_pattern)
-                    if not replace_txt:
-                        replace_txt = ''
-                    print 'Replacing pattern \'{0}\' with text \'{1}\''.format(search_pattern, replace_txt)
-                    rules.insert(0, (search_re, replace_txt))
-                except Exception as e:
-                    self.log.error('Failed to parse %r regexp because %s' %
-                            (search, as_unicode(e)))
+            try:
+                search_re = re.compile(search_pattern)
+                if not replace_txt:
+                    replace_txt = ''
+                rules.insert(0, (search_re, replace_txt))
+            except Exception as e:
+                self.log.error('Failed to parse %r regexp because %s' %
+                        (search, as_unicode(e)))

-        #search / replace using the sr?_search / sr?_replace options
-        for search, replace in [['sr3_search', 'sr3_replace'], ['sr2_search', 'sr2_replace'], ['sr1_search', 'sr1_replace']]:
+        # search / replace using the sr?_search / sr?_replace options
+        for i in range(1, 4):
+            search, replace = 'sr%d_search'%i, 'sr%d_replace'%i
            search_pattern = getattr(self.extra_opts, search, '')
            replace_txt = getattr(self.extra_opts, replace, '')
-            do_search_replace(search_pattern, replace_txt)
+            if search_pattern:
+                do_search_replace(search_pattern, replace_txt)

        # multi-search / replace using the search_replace option
-        search_replace = search_replace_option.decode(getattr(self.extra_opts, 'search_replace', '[]'))
-        for search_pattern, replace_txt in search_replace:
-            do_search_replace(search_pattern, replace_txt)
-            
+        search_replace = getattr(self.extra_opts, 'search_replace', None)
+        if search_replace:
+            search_replace = json.loads(search_replace)
+            for search_pattern, replace_txt in search_replace:
+                do_search_replace(search_pattern, replace_txt)
+
        end_rules = []
        # delete soft hyphens - moved here so it's executed after header/footer removal
        if is_pdftohtml:
--- a/src/calibre/ebooks/conversion/search_replace_option.py
+++ b/src/calibre/ebooks/conversion/search_replace_option.py
@ -1,50 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-
-__license__   = 'GPL v3'
-__copyright__ = '2012, Eli Algranti <idea00@hotmail.com>'
-__docformat__ = 'restructuredtext en'
-
-import json
-from itertools import izip
-
-def encodeJson(definition):
-    '''
-    Encode a search/replace definition using json.
-    '''
-    return 'json:' + json.dumps(definition)
-
-def encodeFile(definition, filename):
-    '''
-    Encode a search/replace definition into a file
-    '''
-    with open(filename, 'w') as f:
-        for search,replace in definition:
-            f.write(search + '\n')
-            f.write(replace + '\n')
-
-    return 'file:'+filename
-
-
-def decode(definition):
-    '''
-    Decodes a search/replace definition
-    '''
-    if definition.startswith('json:'):
-        return json.loads(definition[len('json:'):])
-    elif definition.startswith('file:'):
-        with open(definition[len('file:'):], 'r') as f:
-            ans = []
-            for search, replace in izip(f, f):
-                ans.append([search.rstrip('\n\r'), replace.rstrip('\n\r')])
-        return ans
-    raise Exception('Invalid definition')
-
-            
-            
-
-
-
-
-    
-
--- a/src/calibre/ebooks/metadata/sources/worker.py
+++ b/src/calibre/ebooks/metadata/sources/worker.py
@ -48,7 +48,8 @@ def merge_result(oldmi, newmi, ensure_fields=None):

    return newmi

-def main(do_identify, covers, metadata, ensure_fields):
+def main(do_identify, covers, metadata, ensure_fields, tdir):
+    os.chdir(tdir)
    failed_ids = set()
    failed_covers = set()
    all_failed = True
@ -103,7 +104,8 @@ def single_identify(title, authors, identifiers):
    return [metadata_to_opf(r) for r in results], [r.has_cached_cover_url for
        r in results], dump_caches(), log.dump()

-def single_covers(title, authors, identifiers, caches):
+def single_covers(title, authors, identifiers, caches, tdir):
+    os.chdir(tdir)
    load_caches(caches)
    log = GUILog()
    results = Queue()
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@ -308,8 +308,10 @@ class MOBIHeader(object): # {{{
        self.extra_data_flags = 0
        if self.has_extra_data_flags:
            self.unknown4 = self.raw[180:192]
-            self.fdst_idx, self.fdst_count = struct.unpack_from(b'>II',
+            self.fdst_idx, self.fdst_count = struct.unpack_from(b'>LL',
                    self.raw, 192)
+            if self.fdst_count <= 1:
+                self.fdst_idx = NULL_INDEX
            (self.fcis_number, self.fcis_count, self.flis_number,
                    self.flis_count) = struct.unpack(b'>IIII',
                            self.raw[200:216])
@ -342,7 +344,7 @@ class MOBIHeader(object): # {{{
                'first_non_book_record', 'datp_record_offset', 'fcis_number',
                'flis_number', 'primary_index_record', 'fdst_idx',
                'first_image_index'):
-            if hasattr(self, x):
+            if hasattr(self, x) and getattr(self, x) != NULL_INDEX:
                setattr(self, x, self.header_offset+getattr(self, x))

        if self.has_exth:
--- a/src/calibre/ebooks/mobi/debug/index.py
+++ b/src/calibre/ebooks/mobi/debug/index.py
@ -0,0 +1,158 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from collections import OrderedDict, namedtuple
+
+from calibre.ebooks.mobi.reader.headers import NULL_INDEX
+from calibre.ebooks.mobi.reader.index import (CNCX, parse_indx_header,
+        parse_tagx_section, parse_index_record, INDEX_HEADER_FIELDS)
+from calibre.ebooks.mobi.reader.ncx import (tag_fieldname_map, default_entry)
+
+File = namedtuple('File',
+    'file_number name divtbl_count start_position length')
+
+Elem = namedtuple('Elem',
+    'insert_pos toc_text file_number sequence_number start_pos '
+    'length')
+
+def read_index(sections, idx, codec):
+    table, cncx = OrderedDict(), CNCX([], codec)
+
+    data = sections[idx].raw
+
+    indx_header = parse_indx_header(data)
+    indx_count = indx_header['count']
+
+    if indx_header['ncncx'] > 0:
+        off = idx + indx_count + 1
+        cncx_records = [x.raw for x in sections[off:off+indx_header['ncncx']]]
+        cncx = CNCX(cncx_records, codec)
+
+    tag_section_start = indx_header['tagx']
+    control_byte_count, tags = parse_tagx_section(data[tag_section_start:])
+
+    for i in xrange(idx + 1, idx + 1 + indx_count):
+        # Index record
+        data = sections[i].raw
+        parse_index_record(table, data, control_byte_count, tags, codec,
+                indx_header['ordt_map'], strict=True)
+    return table, cncx, indx_header
+
+class Index(object):
+
+    def __init__(self, idx, records, codec):
+        self.table = self.cncx = self.header = self.records = None
+        if idx != NULL_INDEX:
+            self.table, self.cncx, self.header = read_index(records, idx, codec)
+
+    def render(self):
+        ans = ['*'*10 + ' Index Header ' + '*'*10]
+        a = ans.append
+        if self.header is not None:
+            for field in INDEX_HEADER_FIELDS:
+                a('%-12s: %r'%(field, self.header[field]))
+        ans.extend(['', ''])
+
+        if self.cncx:
+            a('*'*10 + ' CNCX ' + '*'*10)
+            for offset, val in self.cncx.iteritems():
+                a('%10s: %s'%(offset, val))
+            ans.extend(['', ''])
+
+        if self.table is not None:
+            a('*'*10 + ' %d Index Entries '%len(self.table) + '*'*10)
+            for k, v in self.table.iteritems():
+                a('%s: %r'%(k, v))
+
+        if self.records:
+            ans.extend(['', '', '*'*10 + ' Parsed Entries ' + '*'*10])
+            for f in self.records:
+                a(repr(f))
+
+        return ans + ['']
+
+    def __str__(self):
+        return '\n'.join(self.render())
+
+class SKELIndex(Index):
+
+    def __init__(self, skelidx, records, codec):
+        super(SKELIndex, self).__init__(skelidx, records, codec)
+        self.records = []
+
+        if self.table is not None:
+            for i, text in enumerate(self.table.iterkeys()):
+                tag_map = self.table[text]
+                if set(tag_map.iterkeys()) != {1, 6}:
+                    raise ValueError('SKEL Index has unknown tags: %s'%
+                            (set(tag_map.iterkeys())-{1,6}))
+                self.records.append(File(
+                    i, # file_number
+                    text, # name
+                    tag_map[1][0], # divtbl_count
+                    tag_map[6][0], # start_pos
+                    tag_map[6][1]) # length
+                )
+
+class SECTIndex(Index):
+
+    def __init__(self, sectidx, records, codec):
+        super(SECTIndex, self).__init__(sectidx, records, codec)
+        self.records = []
+
+        if self.table is not None:
+             for i, text in enumerate(self.table.iterkeys()):
+                tag_map = self.table[text]
+                if set(tag_map.iterkeys()) != {2, 3, 4, 6}:
+                    raise ValueError('SECT Index has unknown tags: %s'%
+                            (set(tag_map.iterkeys())-{2, 3, 4, 6}))
+
+                toc_text = self.cncx[tag_map[2][0]]
+                self.records.append(Elem(
+                    int(text), # insert_pos
+                    toc_text, # toc_text
+                    tag_map[3][0], # file_number
+                    tag_map[4][0], # sequence_number
+                    tag_map[6][0], # start_pos
+                    tag_map[6][1]  # length
+                    )
+                )
+
+class NCXIndex(Index):
+
+    def __init__(self, ncxidx, records, codec):
+        super(NCXIndex, self).__init__(ncxidx, records, codec)
+        self.records = []
+
+        if self.table is not None:
+            for num, x in enumerate(self.table.iteritems()):
+                text, tag_map = x
+                entry = default_entry.copy()
+                entry['name'] = text
+                entry['num'] = num
+
+                for tag in tag_fieldname_map.iterkeys():
+                    fieldname, i = tag_fieldname_map[tag]
+                    if tag in tag_map:
+                        fieldvalue = tag_map[tag][i]
+                        if tag == 6:
+                            # Appears to be an idx into the KF8 elems table with an
+                            # offset
+                            fieldvalue = tuple(tag_map[tag])
+                        entry[fieldname] = fieldvalue
+                        for which, name in {3:'text', 5:'kind', 70:'description',
+                                71:'author', 72:'image_caption',
+                                73:'image_attribution'}.iteritems():
+                            if tag == which:
+                                entry[name] = self.cncx.get(fieldvalue,
+                                        default_entry[name])
+                self.records.append(entry)
+
+
+
--- a/src/calibre/ebooks/mobi/debug/mobi6.py
+++ b/src/calibre/ebooks/mobi/debug/mobi6.py
@ -393,7 +393,7 @@ class IndexRecord(object): # {{{

            parse_index_record(table, record.raw,
                    index_header.tagx_control_byte_count, tags,
-                    index_header.index_encoding, strict=True)
+                    index_header.index_encoding, {}, strict=True)

        self.indices = []

--- a/src/calibre/ebooks/mobi/debug/mobi8.py
+++ b/src/calibre/ebooks/mobi/debug/mobi8.py
@ -7,10 +7,42 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'

-import sys, os, imghdr
+import sys, os, imghdr, struct
+from itertools import izip

 from calibre.ebooks.mobi.debug.headers import TextRecord
+from calibre.ebooks.mobi.debug.index import (SKELIndex, SECTIndex, NCXIndex)
 from calibre.ebooks.mobi.utils import read_font_record
+from calibre.ebooks.mobi.debug import format_bytes
+from calibre.ebooks.mobi.reader.headers import NULL_INDEX
+
+class FDST(object):
+
+    def __init__(self, raw):
+        if raw[:4] != b'FDST':
+            raise ValueError('KF8 does not have a valid FDST record')
+        self.sec_off, self.num_sections = struct.unpack_from(b'>LL', raw, 4)
+        if self.sec_off != 12:
+            raise ValueError('FDST record has unknown extra fields')
+        secf = b'>%dL' % (self.num_sections*2)
+        secs = struct.unpack_from(secf, raw, self.sec_off)
+        rest = raw[self.sec_off+struct.calcsize(secf):]
+        if rest:
+            raise ValueError('FDST record has trailing data: '
+                    '%s'%format_bytes(rest))
+        self.sections = tuple(izip(secs[::2], secs[1::2]))
+
+    def __str__(self):
+        ans = ['FDST record']
+        a = lambda k, v:ans.append('%s: %s'%(k, v))
+        a('Offset to sections', self.sec_off)
+        a('Number of section records', self.num_sections)
+        ans.append('**** %d Sections ****'% len(self.sections))
+        for sec in self.sections:
+            ans.append('Start: %20d End: %d'%sec)
+
+        return '\n'.join(ans)
+

 class MOBIFile(object):

@ -31,7 +63,10 @@ class MOBIFile(object):
                first_text_record+offset+h8.number_of_text_records])]

        self.raw_text = b''.join(r.raw for r in self.text_records)
+        self.header = self.mf.mobi8_header
        self.extract_resources()
+        self.read_fdst()
+        self.read_indices()

    def print_header(self, f=sys.stdout):
        print (str(self.mf.palmdb).encode('utf-8'), file=f)
@ -43,6 +78,23 @@ class MOBIFile(object):
        print (file=f)
        print (str(self.mf.mobi8_header).encode('utf-8'), file=f)

+    def read_fdst(self):
+        self.fdst = None
+
+        if self.header.fdst_idx != NULL_INDEX:
+            idx = self.header.fdst_idx
+            self.fdst = FDST(self.mf.records[idx].raw)
+            if self.fdst.num_sections != self.header.fdst_count:
+                raise ValueError('KF8 Header contains invalid FDST count')
+
+    def read_indices(self):
+        self.skel_index = SKELIndex(self.header.skel_idx, self.mf.records,
+                self.header.encoding)
+        self.sect_index = SECTIndex(self.header.sect_idx, self.mf.records,
+                self.header.encoding)
+        self.ncx_index = NCXIndex(self.header.primary_index_record,
+                self.mf.records, self.header.encoding)
+
    def extract_resources(self):
        self.resource_map = []
        known_types = {b'FLIS', b'FCIS', b'SRCS',
@ -96,7 +148,19 @@ def inspect_mobi(mobi_file, ddir):
        rec.dump(os.path.join(ddir, 'text_records'))

    for href, payload in f.resource_map:
-        with open(os.path.join(ddir, href), 'wb') as f:
-            f.write(payload)
+        with open(os.path.join(ddir, href), 'wb') as fo:
+            fo.write(payload)

+    if f.fdst:
+        with open(os.path.join(ddir, 'fdst.record'), 'wb') as fo:
+            fo.write(str(f.fdst).encode('utf-8'))
+
+    with open(os.path.join(ddir, 'skel.record'), 'wb') as fo:
+        fo.write(str(f.skel_index).encode('utf-8'))
+
+    with open(os.path.join(ddir, 'sect.record'), 'wb') as fo:
+        fo.write(str(f.sect_index).encode('utf-8'))
+
+    with open(os.path.join(ddir, 'ncx.record'), 'wb') as fo:
+        fo.write(str(f.ncx_index).encode('utf-8'))

--- a/src/calibre/ebooks/mobi/mobiml.py
+++ b/src/calibre/ebooks/mobi/mobiml.py
@ -473,7 +473,7 @@ class MobiMLizer(object):
        if tag in TABLE_TAGS and self.ignore_tables:
            tag = 'span' if tag == 'td' else 'div'

-        if tag == 'table':
+        if tag in ('table', 'td', 'tr'):
            col = style.backgroundColor
            if col:
                elem.set('bgcolor', col)
--- a/src/calibre/ebooks/mobi/reader/index.py
+++ b/src/calibre/ebooks/mobi/reader/index.py
@ -15,6 +15,12 @@ from calibre.ebooks.mobi.utils import (decint, count_set_bits,

 TagX = namedtuple('TagX', 'tag num_of_values bitmask eof')
 PTagX = namedtuple('PTagX', 'tag value_count value_bytes num_of_values')
+INDEX_HEADER_FIELDS = (
+            'len', 'nul1', 'type', 'gen', 'start', 'count', 'code',
+            'lng', 'total', 'ordt', 'ligt', 'nligt', 'ncncx'
+    ) + tuple('unknown%d'%i for i in xrange(27)) + ('ocnt', 'oentries',
+            'ordt1', 'ordt2', 'tagx')
+

 class InvalidFile(ValueError):
    pass
@ -36,11 +42,7 @@ def format_bytes(byts):

 def parse_indx_header(data):
    check_signature(data, b'INDX')
-    words = (
-            'len', 'nul1', 'type', 'gen', 'start', 'count', 'code',
-            'lng', 'total', 'ordt', 'ligt', 'nligt', 'ncncx'
-    ) + tuple('unknown%d'%i for i in xrange(27)) + ('ocnt', 'oentries',
-            'ordt1', 'ordt2', 'tagx')
+    words = INDEX_HEADER_FIELDS
    num = len(words)
    values = struct.unpack(bytes('>%dL' % num), data[4:4*(num+1)])
    ans = dict(zip(words, values))
@ -109,6 +111,12 @@ class CNCX(object): # {{{

    def get(self, offset, default=None):
        return self.records.get(offset, default)
+
+    def __bool__(self):
+        return bool(self.records)
+
+    def iteritems(self):
+        return self.records.iteritems()
 # }}}

 def parse_tagx_section(data):
--- a/src/calibre/ebooks/mobi/reader/mobi8.py
+++ b/src/calibre/ebooks/mobi/reader/mobi8.py
@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'

 import struct, re, os, imghdr
 from collections import namedtuple
-from itertools import repeat
+from itertools import repeat, izip
 from urlparse import urldefrag

 from lxml import etree
@ -71,16 +71,16 @@ class Mobi8Reader(object):
        return self.write_opf(guide, ncx, spine, resource_map)

    def read_indices(self):
-        self.flow_table = (0, NULL_INDEX)
+        self.flow_table = ()

        if self.header.fdstidx != NULL_INDEX:
            header = self.kf8_sections[self.header.fdstidx][0]
            if header[:4] != b'FDST':
                raise ValueError('KF8 does not have a valid FDST record')
-            num_sections, = struct.unpack_from(b'>L', header, 0x08)
-            sections = header[0x0c:]
-            self.flow_table = struct.unpack_from(b'>%dL' % (num_sections*2),
-                    sections, 0)[::2] + (NULL_INDEX,)
+            sec_start, num_sections = struct.unpack_from(b'>LL', header, 4)
+            secs = struct.unpack_from(b'>%dL' % (num_sections*2),
+                    header, sec_start)
+            self.flow_table = tuple(izip(secs[::2], secs[1::2]))

        self.files = []
        if self.header.skelidx != NULL_INDEX:
@ -127,13 +127,10 @@ class Mobi8Reader(object):
        raw_ml = self.mobi6_reader.mobi_html
        self.flows = []
        self.flowinfo = []
+        ft = self.flow_table if self.flow_table else [(0, len(raw_ml))]

        # now split the raw_ml into its flow pieces
-        for j in xrange(0, len(self.flow_table)-1):
-            start = self.flow_table[j]
-            end = self.flow_table[j+1]
-            if end == NULL_INDEX:
-                end = len(raw_ml)
+        for start, end in ft:
            self.flows.append(raw_ml[start:end])

        # the first piece represents the xhtml text
--- a/src/calibre/ebooks/mobi/writer8/init.py
+++ b/src/calibre/ebooks/mobi/writer8/init.py
@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+
--- a/src/calibre/ebooks/mobi/writer8/main.py
+++ b/src/calibre/ebooks/mobi/writer8/main.py
@ -0,0 +1,15 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+class KF8Writer(object):
+
+    def __init__(self, oeb, opts):
+        self.oeb, self.opts, self.log = oeb, opts, oeb.log
+
--- a/src/calibre/ebooks/pdb/haodoo/init.py
+++ b/src/calibre/ebooks/pdb/haodoo/init.py
@ -0,0 +1,11 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@ -641,6 +641,26 @@ def choose_files(window, name, title,
        return fd.get_files()
    return None

+def choose_save_file(window, name, title, filters=[], all_files=True):
+    '''
+    Ask user to choose a file to save to. Can be a non-existent file.
+    :param filters: list of allowable extensions. Each element of the list
+                     must be a 2-tuple with first element a string describing
+                     the type of files to be filtered and second element a list
+                     of extensions.
+    :param all_files: If True add All files to filters.
+    '''
+    mode = QFileDialog.AnyFile
+    fd = FileDialog(title=title, name=name, filters=filters,
+                    parent=window, add_all_files_filter=all_files, mode=mode)
+    fd.setParent(None)
+    ans = None
+    if fd.accepted:
+        ans = fd.get_files()
+        if ans:
+            ans = ans[0]
+    return ans
+
 def choose_images(window, name, title, select_only_single_file=True):
    mode = QFileDialog.ExistingFile if select_only_single_file else QFileDialog.ExistingFiles
    fd = FileDialog(title=title, name=name,
--- a/src/calibre/gui2/convert/init.py
+++ b/src/calibre/gui2/convert/init.py
@ -252,6 +252,7 @@ class Widget(QWidget):


    def set_value_handler(self, g, val):
+        'Return True iff you handle setting the value for g'
        return False

    def post_set_value(self, g, val):
--- a/src/calibre/gui2/convert/regex_builder.py
+++ b/src/calibre/gui2/convert/regex_builder.py
@ -129,6 +129,8 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
            d.exec_()
            if d.result() == QDialog.Accepted:
                format = d.format()
+            else:
+                return False

        if not format:
            error_dialog(self, _('No formats available'),
@ -240,5 +242,8 @@ class RegexEdit(QWidget, Ui_Edit):
    def regex(self):
        return self.text

+    def clear(self):
+        self.edit.clear()
+
    def check(self):
        return True
--- a/src/calibre/gui2/convert/search_and_replace.py
+++ b/src/calibre/gui2/convert/search_and_replace.py
@ -4,14 +4,15 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>, 2012 Eli Algranti <idea00@hotmail.com>'
 __docformat__ = 'restructuredtext en'

-import re
-from calibre.ebooks.conversion import search_replace_option
+import re, codecs, json
+
+from PyQt4.Qt import Qt, QTableWidgetItem

-from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QTableWidget, QTableWidgetItem, QFileDialog, QMessageBox
 from calibre.gui2.convert.search_and_replace_ui import Ui_Form
 from calibre.gui2.convert import Widget
-from calibre.gui2 import error_dialog
+from calibre.gui2 import (error_dialog, question_dialog, choose_files,
+        choose_save_file)
+from calibre import as_unicode

 class SearchAndReplaceWidget(Widget, Ui_Form):

@ -22,6 +23,15 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
    STRIP_TEXT_FIELDS = False

    def __init__(self, parent, get_option, get_help, db=None, book_id=None):
+        # Dummy attributes to fool the Widget() option handler code. We handle
+        # everything in our *handler methods.
+        for i in range(1, 4):
+            x = 'sr%d_'%i
+            for y in ('search', 'replace'):
+                z = x + y
+                setattr(self, 'opt_'+z, z)
+        self.opt_search_replace = 'search_replace'
+
        Widget.__init__(self, parent,
                ['search_replace',
                 'sr1_search', 'sr1_replace',
@ -42,14 +52,15 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
        self.search_replace.setColumnCount(2)
        self.search_replace.setColumnWidth(0, 300)
        self.search_replace.setColumnWidth(1, 300)
-        self.search_replace.setHorizontalHeaderLabels([_('Search Regular Expression'), _('Replacement Text')])
+        self.search_replace.setHorizontalHeaderLabels([
+            _('Search Regular Expression'), _('Replacement Text')])

-        self.connect(self.sr_add, SIGNAL('clicked()'), self.sr_add_clicked)
-        self.connect(self.sr_change, SIGNAL('clicked()'), self.sr_change_clicked)
-        self.connect(self.sr_remove, SIGNAL('clicked()'), self.sr_remove_clicked)
-        self.connect(self.sr_load, SIGNAL('clicked()'), self.sr_load_clicked)
-        self.connect(self.sr_save, SIGNAL('clicked()'), self.sr_save_clicked)
-        self.connect(self.search_replace, SIGNAL('currentCellChanged(int, int, int, int)'), self.sr_currentCellChanged)
+        self.sr_add.clicked.connect(self.sr_add_clicked)
+        self.sr_change.clicked.connect(self.sr_change_clicked)
+        self.sr_remove.clicked.connect(self.sr_remove_clicked)
+        self.sr_load.clicked.connect(self.sr_load_clicked)
+        self.sr_save.clicked.connect(self.sr_save_clicked)
+        self.search_replace.currentCellChanged.connect(self.sr_currentCellChanged)

        self.initialize_options(get_option, get_help, db, book_id)

@ -57,7 +68,7 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
        if self.sr_search.regex:
            row = self.sr_add_row(self.sr_search.regex, self.sr_replace.text())
            self.search_replace.setCurrentCell(row, 0)
-            
+
    def sr_add_row(self, search, replace):
        row = self.search_replace.rowCount()
        self.search_replace.setRowCount(row + 1)
@ -75,23 +86,43 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
            self.search_replace.item(row, 0).setText(self.sr_search.regex)
            self.search_replace.item(row, 1).setText(self.sr_replace.text())
            self.search_replace.setCurrentCell(row, 0)
-            
+
    def sr_remove_clicked(self):
        row = self.search_replace.currentRow()
        if  row >= 0:
            self.search_replace.removeRow(row)
            self.search_replace.setCurrentCell(row-1, 0)
+            self.sr_search.clear()
+            self.sr_replace.clear()

    def sr_load_clicked(self):
-        filename = QFileDialog.getOpenFileName(self, _('Load Calibre Search-Replace definitions file'), '.', _('Calibre Search-Replace definitions file (*.csr)'))
-        if filename:
-            self.set_value_handler(self.opt_search_replace, 'file:'+unicode(filename))
-            
+        files = choose_files(self, 'sr_saved_patterns',
+                _('Load Calibre Search-Replace definitions file'),
+                filters=[
+                    (_('Calibre Search-Replace definitions file'), ['csr'])
+                    ], select_only_single_file=True)
+        if files:
+            from calibre.ebooks.conversion.cli import read_sr_patterns
+            try:
+                self.set_value(self.opt_search_replace,
+                    read_sr_patterns(files[0]))
+            except Exception as e:
+                error_dialog(self, _('Failed to read'),
+                        _('Failed to load patterns from %s, click Show details'
+                            ' to learn more.')%files[0], det_msg=as_unicode(e),
+                        show=True)
+
    def sr_save_clicked(self):
-        filename = QFileDialog.getSaveFileName(self, _('Save Calibre Search-Replace definitions file'), '.', _('Calibre Search-Replace definitions file (*.csr)'))
+        filename = choose_save_file(self, 'sr_saved_patterns',
+                _('Save Calibre Search-Replace definitions file'),
+                filters=[
+                    (_('Calibre Search-Replace definitions file'), ['csr'])
+                    ])
        if filename:
-            search_replace_option.encodeFile(self.get_definitions(), unicode(filename))
-        
+            with codecs.open(filename, 'w', 'utf-8') as f:
+                for search, replace in self.get_definitions():
+                    f.write(search + u'\n' + replace + u'\n\n')
+
    def sr_currentCellChanged(self, row, column, previousRow, previousColumn) :
        if row >= 0:
            self.sr_change.setEnabled(True)
@ -101,7 +132,7 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
        else:
            self.sr_change.setEnabled(False)
            self.sr_remove.setEnabled(False)
-        
+
    def break_cycles(self):
        Widget.break_cycles(self)

@ -119,8 +150,6 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
        self.sr_search.set_doc(doc)

    def pre_commit_check(self):
-
-
        definitions = self.get_definitions()

        # Verify the search/replace in the edit widgets has been
@ -135,16 +164,15 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                if search == edit_search and replace == edit_replace:
                    found = True
                    break
-            if not found:
-                msgBox = QMessageBox(self)
-                msgBox.setText(_('The search / replace definition being edited has not been added to the list of definitions'))
-                msgBox.setInformativeText(_('Do you wish to continue with the conversion (the definition will not be used)?'))
-                msgBox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
-                msgBox.setDefaultButton(QMessageBox.No)
-                if msgBox.exec_() != QMessageBox.Yes:
+            if not found and not question_dialog(self,
+                    _('Unused Search & Replace definition'),
+                    _('The search / replace definition being edited '
+                        ' has not been added to the list of definitions. '
+                        'Do you wish to continue with the conversion '
+                        '(the definition will not be used)?')):
                    return False

-        # Verify all search expressions are valid        
+        # Verify all search expressions are valid
        for search, replace in definitions:
            try:
                re.compile(search)
@ -152,49 +180,19 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                error_dialog(self, _('Invalid regular expression'),
                             _('Invalid regular expression: %s')%err, show=True)
                return False
-        
+
        return True

-    # Options
-    @property
-    def opt_search_replace(self):
-        return 'search_replace'
-
-    @property
-    def opt_sr1_search(self):
-        return 'sr1_search'
-
-    @property
-    def opt_sr1_replace(self):
-        return 'sr1_replace'
-
-    @property
-    def opt_sr2_search(self):
-        return 'sr2_search'
-
-    @property
-    def opt_sr2_replace(self):
-        return 'sr2_replace'
-
-    @property
-    def opt_sr3_search(self):
-        return 'sr3_search'
-
-    @property
-    def opt_sr3_replace(self):
-        return 'sr3_replace'
-
-
    # Options handling

    def connect_gui_obj_handler(self, g, slot):
-        if g == self.opt_search_replace:
+        if g is self.opt_search_replace:
            self.search_replace.cellChanged.connect(slot)

    def get_value_handler(self, g):
-        if g != self.opt_search_replace:
-            return None
-        return search_replace_option.encodeJson(self.get_definitions())
+        if g is self.opt_search_replace:
+            return json.dumps(self.get_definitions())
+        return None

    def get_definitions(self):
        ans = []
@ -206,22 +204,21 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
        return ans

    def set_value_handler(self, g, val):
-        if g != self.opt_search_replace:
-            self.handle_legacy(g, val)
+        if g is not self.opt_search_replace:
            return True

        try:
-            rowItems = search_replace_option.decode(val)
+            rowItems = json.loads(val)
            if not isinstance(rowItems, list):
                rowItems = []
        except:
            rowItems = []

        if len(rowItems) == 0:
-            return True
+            self.search_replace.clearContents()

        self.search_replace.setRowCount(len(rowItems))
-           
+
        for row, colItems in enumerate(rowItems):
            for col, cellValue in enumerate(colItems):
                newItem = self.search_replace.itemPrototype().clone()
@ -229,33 +226,44 @@ class SearchAndReplaceWidget(Widget, Ui_Form):
                self.search_replace.setItem(row,col, newItem)
        return True

-    def handle_legacy(self, g, val):
+    def apply_recommendations(self, recs):
        '''
-        Handles legacy search/replace options sr1_search, sr1_replace,
-        sr2_search, sr2_replace, sr3_search, sr3_replace.
-        Before introducing the search_replace option only three search/replace
-        definitions could be made. These where stored in the options named above.
-        This function is for backward compatibility with saved options and for
-        compatibility with setting sr* options in the CLI.
+        Handle the legacy sr* options that may have been previously saved. They
+        are applied only if the new search_replace option has not been set in
+        recs.
        '''
+        new_val = None
+        legacy = {}
+        rest = {}
+        for name, val in recs.items():
+            if name == 'search_replace':
+                new_val = val
+                if name in getattr(recs, 'disabled_options', []):
+                    self.search_replace.setDisabled(True)
+            elif name.startswith('sr'):
+                legacy[name] = val if val else ''
+            else:
+                rest[name] = val

-        if not val: return
-        
-        row = int(g[2]) - 1 # the row to set in the search_replace table is 0 for sr1_*, 1 for sr2_*, etc
-        col = (0 if g[4] == 's' else 1) # the fourth character in g is 's' for search options and 'r' for replace options
+        if rest:
+            super(SearchAndReplaceWidget, self).apply_recommendations(rest)

-        # add any missing rows 
-        while self.search_replace.rowCount() < row+1:
-            self.sr_add_row('', '')
-        
-        # set the value
-        self.search_replace.item(row, col).setText(val)
+        self.set_value(self.opt_search_replace, None)
+        if new_val is None and legacy:
+            for i in range(1, 4):
+                x = 'sr%d'%i
+                s, r = x+'_search', x+'_replace'
+                s, r = legacy.get(s, ''), legacy.get(r, '')
+                if s:
+                    self.sr_add_row(s, r)
+        if new_val is not None:
+            self.set_value(self.opt_search_replace, new_val)

    def setup_help_handler(self, g, help):
-        if g != self.opt_search_replace:
-            return True
-
-        self.search_replace._help = help
-        self.setup_widget_help(self.search_replace)
+        if g is self.opt_search_replace:
+            self.search_replace._help = _(
+                'The list of search/replace definitions that will be applied '
+                'to this conversion.')
+            self.setup_widget_help(self.search_replace)
        return True
-        
+
--- a/src/calibre/gui2/convert/search_and_replace.ui
+++ b/src/calibre/gui2/convert/search_and_replace.ui
@ -6,7 +6,7 @@
   <rect>
    <x>0</x>
    <y>0</y>
-    <width>468</width>
+    <width>667</width>
    <height>451</height>
   </rect>
  </property>
@ -86,68 +86,62 @@
      <number>0</number>
     </property>
     <item>
-      <widget class="QPushButton" name="sr_add" native="true">
+      <widget class="QPushButton" name="sr_add">
       <property name="text">
        <string>Add</string>
       </property>
-	  </widget>
+      </widget>
     </item>
     <item>
-      <widget class="QPushButton" name="sr_change" native="true">
+      <widget class="QPushButton" name="sr_change">
+       <property name="enabled">
+        <bool>false</bool>
+       </property>
       <property name="text">
        <string>Change</string>
       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="sr_remove">
       <property name="enabled">
        <bool>false</bool>
       </property>
-	  </widget>
-     </item>
-     <item>			
-      <widget class="QPushButton" name="sr_remove" native="true">
       <property name="text">
        <string>Remove</string>
       </property>
-       <property name="enabled">
-        <bool>false</bool>
-       </property>
-	  </widget>
+      </widget>
     </item>
-     <item>			
-      <widget class="QPushButton" name="sr_load" native="true">
+     <item>
+      <widget class="QPushButton" name="sr_load">
       <property name="text">
        <string>Load</string>
       </property>
-	  </widget>
-	 </item>
-     <item>	 
-	  <widget class="QPushButton" name="sr_save" native="true">
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="sr_save">
       <property name="text">
        <string>Save</string>
       </property>
-	  </widget>
+      </widget>
     </item>
-    </layout>	
+    </layout>
   </item>
   <item row="3" column="0">
-	<widget class="QTableWidget" name="search_replace" native="true">
-	 <property name="sizePolicy">
-	  <sizepolicy hsizetype="Minimum" vsizetype="Preferred">
-	   <horstretch>0</horstretch>
-	   <verstretch>0</verstretch>
-	  </sizepolicy>
-	 </property>
-     <property name="selectionBehavior">
-      <enum>QAbstractItemView::SelectRows</enum>
-	 </property>
+    <widget class="QTableWidget" name="search_replace">
     <property name="selectionMode">
      <enum>QAbstractItemView::SingleSelection</enum>
-	 </property>
+     </property>
+     <property name="selectionBehavior">
+      <enum>QAbstractItemView::SelectRows</enum>
+     </property>
    </widget>
   </item>
   <item row="0" column="0">
    <widget class="QLabel" name="label">
     <property name="text">
-      <string>&lt;p&gt;Search and replace uses &lt;i&gt;regular expressions&lt;/i&gt;. See the &lt;a href=&quot;http://manual.calibre-ebook.com/regexp.html&quot;&gt;regular expressions tutorial&lt;/a&gt; to get started with regular expressions. Also clicking the wizard button below will allow you to test your regular expression against the current input document.</string>
+      <string>&lt;p&gt;Search and replace uses &lt;i&gt;regular expressions&lt;/i&gt;. See the &lt;a href=&quot;http://manual.calibre-ebook.com/regexp.html&quot;&gt;regular expressions tutorial&lt;/a&gt; to get started with regular expressions. Also clicking the wizard button below will allow you to test your regular expression against the current input document. When you are happy with an expression, click the Add button to add it to the list of expressions.</string>
     </property>
     <property name="wordWrap">
      <bool>true</bool>
--- a/src/calibre/gui2/dialogs/search.py
+++ b/src/calibre/gui2/dialogs/search.py
@ -173,7 +173,7 @@ class SearchDialog(QDialog, Ui_Dialog):
        self.box_last_values['tags_box'] = tags
        tags = [t.strip() for t in tags.split(',') if t.strip()]
        if tags:
-            tags = ['tags:"=' + t + '"' for t in tags]
+            tags = ['tags:"' + self.mc + t + '"' for t in tags]
            ans.append('(' + ' or '.join(tags) + ')')
        general = unicode(self.general_box.text())
        self.box_last_values['general_box'] = general
--- a/src/calibre/gui2/metadata/bulk_download.py
+++ b/src/calibre/gui2/metadata/bulk_download.py
@ -232,8 +232,8 @@ def download(all_ids, tf, db, do_identify, covers, ensure_fields,
                    metadata.iteritems()}
            try:
                ret = fork_job('calibre.ebooks.metadata.sources.worker', 'main',
-                        (do_identify, covers, metadata, ensure_fields),
-                        cwd=tdir, abort=abort, heartbeat=heartbeat, no_output=True)
+                        (do_identify, covers, metadata, ensure_fields, tdir),
+                        abort=abort, heartbeat=heartbeat, no_output=True)
            except WorkerError as e:
                if e.orig_tb:
                    raise Exception('Failed to download metadata. Original '
--- a/src/calibre/gui2/metadata/single_download.py
+++ b/src/calibre/gui2/metadata/single_download.py
@ -573,8 +573,9 @@ class CoverWorker(Thread): # {{{
            try:
                res = fork_job('calibre.ebooks.metadata.sources.worker',
                    'single_covers',
-                    (self.title, self.authors, self.identifiers, self.caches),
-                    cwd=tdir, no_output=True, abort=self.abort)
+                    (self.title, self.authors, self.identifiers, self.caches,
+                        tdir),
+                    no_output=True, abort=self.abort)
                self.log.append_dump(res['result'])
            finally:
                self.keep_going = False
--- a/src/calibre/manual/news.rst
+++ b/src/calibre/manual/news.rst
@ -276,6 +276,7 @@ Once the download is complete, you can look at the downloaded :term:`HTML` by op

 If you're satisfied with your recipe, and you feel there is enough demand to justify its inclusion into the set of built-in recipes, post your recipe in the `calibre recipes forum <http://www.mobileread.com/forums/forumdisplay.php?f=228>`_ to share it with other calibre users.

+.. note:: On OS X, the ebook-convert command will not be available by default. Go to Preferences->Miscellaneous and click the install command line tools button to make it available.

 .. seealso::

--- a/src/calibre/translations/af.po
+++ b/src/calibre/translations/af.po
--- a/src/calibre/translations/ar.po
+++ b/src/calibre/translations/ar.po
--- a/src/calibre/translations/ast.po
+++ b/src/calibre/translations/ast.po
--- a/src/calibre/translations/az.po
+++ b/src/calibre/translations/az.po
--- a/src/calibre/translations/bg.po
+++ b/src/calibre/translations/bg.po
--- a/src/calibre/translations/bn.po
+++ b/src/calibre/translations/bn.po
--- a/src/calibre/translations/br.po
+++ b/src/calibre/translations/br.po
--- a/src/calibre/translations/bs.po
+++ b/src/calibre/translations/bs.po
--- a/src/calibre/translations/ca.po
+++ b/src/calibre/translations/ca.po
--- a/src/calibre/translations/calibre.pot
+++ b/src/calibre/translations/calibre.pot
@ -4,9 +4,9 @@
 #
 msgid ""
 msgstr ""
-"Project-Id-Version: calibre 0.8.46\n"
-"POT-Creation-Date: 2012-04-08 15:08+IST\n"
-"PO-Revision-Date: 2012-04-08 15:08+IST\n"
+"Project-Id-Version: calibre 0.8.47\n"
+"POT-Creation-Date: 2012-04-13 09:24+IST\n"
+"PO-Revision-Date: 2012-04-13 09:24+IST\n"
 "Last-Translator: Automatically generated\n"
 "Language-Team: LANGUAGE\n"
 "MIME-Version: 1.0\n"
@ -38,7 +38,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/devices/prs505/sony_cache.py:661
 #: /home/kovid/work/calibre/src/calibre/devices/prst1/driver.py:337
 #: /home/kovid/work/calibre/src/calibre/devices/prst1/driver.py:338
-#: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:489
+#: /home/kovid/work/calibre/src/calibre/devices/usbms/driver.py:493
 #: /home/kovid/work/calibre/src/calibre/ebooks/chm/metadata.py:57
 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:109
 #: /home/kovid/work/calibre/src/calibre/ebooks/conversion/plugins/chm_input.py:112
@ -78,7 +78,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/mobi.py:472
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/opf2.py:1134
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/opf2.py:1245
-#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/pdb.py:41
+#: /home/kovid/work/calibre/src/calibre/ebooks/metadata/pdb.py:44
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/pdf.py:29
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/plucker.py:25
 #: /home/kovid/work/calibre/src/calibre/ebooks/metadata/pml.py:23
@ -4010,6 +4010,10 @@ msgstr ""
 msgid "Generating %s catalog..."
 msgstr ""

+#: /home/kovid/work/calibre/src/calibre/gui2/actions/catalog.py:71
+msgid "Catalog generation complete, with warnings."
+msgstr ""
+
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/catalog.py:86
 msgid "Catalog generated."
 msgstr ""
@ -4128,7 +4132,7 @@ msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/actions/choose_library.py:283
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/plugin_updater.py:726
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:201
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:204
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/plugins.py:308
 msgid "Already exists"
 msgstr ""
@ -4343,7 +4347,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/plugin_updater.py:674
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/restore_library.py:93
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/tag_list_editor.py:216
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:371
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:374
 #: /home/kovid/work/calibre/src/calibre/gui2/jobs.py:597
 #: /home/kovid/work/calibre/src/calibre/gui2/jobs.py:607
 #: /home/kovid/work/calibre/src/calibre/gui2/preferences/columns.py:102
@ -4590,7 +4594,7 @@ msgstr ""
 #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:101
 #: /home/kovid/work/calibre/src/calibre/gui2/dnd.py:84
 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:507
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:817
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:818
 msgid "Download failed"
 msgstr ""

@ -4618,7 +4622,7 @@ msgid "Download complete"
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/actions/edit_metadata.py:121
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:879
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:880
 msgid "Download log"
 msgstr ""

@ -8107,15 +8111,15 @@ msgid "Copied"
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:141
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:872
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:873
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:205
 msgid "Copy to clipboard"
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:189
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/message_box.py:244
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:936
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1042
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:937
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1043
 msgid "View log"
 msgstr ""

@ -9708,90 +9712,98 @@ msgstr ""
 msgid "&Preview {0}"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:141
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:112
+msgid "No recipes"
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:113
+msgid "No custom recipes created."
+msgstr ""
+
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:144
 msgid "No recipe selected"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:146
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:149
 #, python-format
 msgid "The attached file: %(fname)s is a recipe to download %(title)s."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:149
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:152
 msgid "Recipe for "
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:166
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:177
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:169
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:180
 #: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles_ui.py:265
 msgid "Switch to Advanced mode"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:172
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:180
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:175
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:183
 msgid "Switch to Basic mode"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:190
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:193
 msgid "Feed must have a title"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:191
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:194
 msgid "The feed must have a title"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:195
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:198
 msgid "Feed must have a URL"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:196
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:199
 #, python-format
 msgid "The feed %s must have a URL"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:202
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:205
 msgid "This feed has already been added to the recipe"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:244
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:253
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:340
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:247
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:256
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:343
 msgid "Invalid input"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:245
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:254
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:341
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:248
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:257
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:344
 #, python-format
 msgid "<p>Could not create recipe. Error:<br>%s"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:258
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:317
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:344
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:261
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:320
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:347
 msgid "Replace recipe?"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:259
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:318
-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:345
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:262
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:321
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:348
 #, python-format
 msgid "A custom recipe named %s already exists. Do you want to replace it?"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:285
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:288
 msgid "Choose builtin recipe"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:331
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:334
 msgid "Choose a recipe file"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:332
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:335
 msgid "Recipes"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:372
+#: /home/kovid/work/calibre/src/calibre/gui2/dialogs/user_profiles.py:375
 msgid "You will lose any unsaved changes. To save your changes, click the Add/Update recipe button. Continue?"
 msgstr ""

@ -10524,7 +10536,7 @@ msgid "Previous Page"
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/lrf_renderer/main_ui.py:133
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:933
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:934
 #: /home/kovid/work/calibre/src/calibre/gui2/store/web_store_dialog_ui.py:62
 #: /home/kovid/work/calibre/src/calibre/gui2/viewer/main_ui.py:193
 msgid "Back"
@ -10975,7 +10987,7 @@ msgid "Edit Metadata"
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/gui2/metadata/single.py:66
-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:926
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:927
 #: /home/kovid/work/calibre/src/calibre/library/server/browse.py:107
 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:219
 #: /home/kovid/work/calibre/src/calibre/web/feeds/templates.py:410
@ -11143,38 +11155,38 @@ msgstr ""
 msgid "Failed to find any books that match your search. Try making the search <b>less specific</b>. For example, use only the author's last name and a single distinctive word from the title.<p>To see the full log, click Show Details."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:624
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:625
 msgid "Current cover"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:627
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:628
 msgid "Searching..."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:787
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:788
 #, python-format
 msgid "Downloading covers for <b>%s</b>, please wait..."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:818
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:819
 msgid "Failed to download any covers, click \"Show details\" for details."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:824
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:825
 #, python-format
 msgid "Could not find any covers for <b>%s</b>"
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:826
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:827
 #, python-format
 msgid "Found <b>%(num)d</b> covers of %(title)s. Pick the one you like best."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:915
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:916
 msgid "Downloading metadata..."
 msgstr ""

-#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1026
+#: /home/kovid/work/calibre/src/calibre/gui2/metadata/single_download.py:1027
 msgid "Downloading cover..."
 msgstr ""

@ -16461,7 +16473,7 @@ msgid "When searching for text without using lookup prefixes, as for example, Re
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/utils/config_base.py:420
-msgid "Choose columns to be searched when not using prefixes, as for example, when searching for Redd instead of title:Red. Enter a list of search/lookup names separated by commas. Only takes effect if you set the option to limit search columns above."
+msgid "Choose columns to be searched when not using prefixes, as for example, when searching for Red instead of title:Red. Enter a list of search/lookup names separated by commas. Only takes effect if you set the option to limit search columns above."
 msgstr ""

 #: /home/kovid/work/calibre/src/calibre/utils/formatter.py:31
--- a/src/calibre/translations/cs.po
+++ b/src/calibre/translations/cs.po
--- a/src/calibre/translations/cy.po
+++ b/src/calibre/translations/cy.po
--- a/src/calibre/translations/da.po
+++ b/src/calibre/translations/da.po
--- a/src/calibre/translations/de.po
+++ b/src/calibre/translations/de.po
--- a/src/calibre/translations/el.po
+++ b/src/calibre/translations/el.po
--- a/src/calibre/translations/en_AU.po
+++ b/src/calibre/translations/en_AU.po
--- a/src/calibre/translations/en_CA.po
+++ b/src/calibre/translations/en_CA.po
--- a/src/calibre/translations/en_GB.po
+++ b/src/calibre/translations/en_GB.po
--- a/src/calibre/translations/eo.po
+++ b/src/calibre/translations/eo.po
--- a/src/calibre/translations/es.po
+++ b/src/calibre/translations/es.po
--- a/src/calibre/translations/et.po
+++ b/src/calibre/translations/et.po
--- a/src/calibre/translations/eu.po
+++ b/src/calibre/translations/eu.po
--- a/src/calibre/translations/fa.po
+++ b/src/calibre/translations/fa.po
--- a/src/calibre/translations/fi.po
+++ b/src/calibre/translations/fi.po
--- a/src/calibre/translations/fo.po
+++ b/src/calibre/translations/fo.po
--- a/src/calibre/translations/fr.po
+++ b/src/calibre/translations/fr.po
--- a/src/calibre/translations/fr_CA.po
+++ b/src/calibre/translations/fr_CA.po
--- a/src/calibre/translations/gl.po
+++ b/src/calibre/translations/gl.po
--- a/src/calibre/translations/gu.po
+++ b/src/calibre/translations/gu.po
--- a/src/calibre/translations/he.po
+++ b/src/calibre/translations/he.po
--- a/src/calibre/translations/hi.po
+++ b/src/calibre/translations/hi.po
--- a/src/calibre/translations/hr.po
+++ b/src/calibre/translations/hr.po
--- a/src/calibre/translations/hu.po
+++ b/src/calibre/translations/hu.po
--- a/src/calibre/translations/id.po
+++ b/src/calibre/translations/id.po
--- a/src/calibre/translations/is.po
+++ b/src/calibre/translations/is.po
--- a/src/calibre/translations/it.po
+++ b/src/calibre/translations/it.po
--- a/src/calibre/translations/ja.po
+++ b/src/calibre/translations/ja.po
--- a/src/calibre/translations/kn.po
+++ b/src/calibre/translations/kn.po
--- a/src/calibre/translations/ko.po
+++ b/src/calibre/translations/ko.po
--- a/src/calibre/translations/ku.po
+++ b/src/calibre/translations/ku.po
--- a/src/calibre/translations/lt.po
+++ b/src/calibre/translations/lt.po
--- a/src/calibre/translations/ltg.po
+++ b/src/calibre/translations/ltg.po
--- a/src/calibre/translations/lv.po
+++ b/src/calibre/translations/lv.po
--- a/src/calibre/translations/mk.po
+++ b/src/calibre/translations/mk.po
--- a/src/calibre/translations/ml.po
+++ b/src/calibre/translations/ml.po
--- a/src/calibre/translations/mr.po
+++ b/src/calibre/translations/mr.po
--- a/src/calibre/translations/ms.po
+++ b/src/calibre/translations/ms.po
--- a/src/calibre/translations/nb.po
+++ b/src/calibre/translations/nb.po
--- a/src/calibre/translations/nds.po
+++ b/src/calibre/translations/nds.po
--- a/src/calibre/translations/nl.po
+++ b/src/calibre/translations/nl.po
--- a/src/calibre/translations/nn.po
+++ b/src/calibre/translations/nn.po
--- a/src/calibre/translations/oc.po
+++ b/src/calibre/translations/oc.po
--- a/src/calibre/translations/pa.po
+++ b/src/calibre/translations/pa.po
--- a/src/calibre/translations/pl.po
+++ b/src/calibre/translations/pl.po
--- a/src/calibre/translations/pt.po
+++ b/src/calibre/translations/pt.po
--- a/src/calibre/translations/pt_BR.po
+++ b/src/calibre/translations/pt_BR.po
--- a/src/calibre/translations/ro.po
+++ b/src/calibre/translations/ro.po
--- a/src/calibre/translations/ru.po
+++ b/src/calibre/translations/ru.po
--- a/src/calibre/translations/sc.po
+++ b/src/calibre/translations/sc.po
--- a/src/calibre/translations/si.po
+++ b/src/calibre/translations/si.po
--- a/src/calibre/translations/sk.po
+++ b/src/calibre/translations/sk.po
--- a/src/calibre/translations/sl.po
+++ b/src/calibre/translations/sl.po
--- a/Show More
+++ b/Show More