Merge branch 'master' of https://github.com/kovidgoyal/calibre

Updating source to 1.29

Changelog.yaml

@@ -20,6 +20,92 @@
 #   new recipes:
 #     - title: 
 
+- version: 1.29.0
+  date: 2014-03-21
+
+  new features:
+    - title: "Edit Book: Add support for saved searches. Click Search->Saved Searches to bring up a dialog where you can create and manage saved searches"
+
+    - title: "Edit Book: New tool to specify semantics in EPUB books (semantics are items in the guide such as preface, title-page, dedication, etc.). To use it, go to Tools->Set Semantics"
+      tickets: [1287025]
+
+    - title: "Edit Book: Preview panel: Add a copy selected text action to the context menu"
+
+    - title: "Edit Book: When inserting hyperlinks, allow specifying the text for the hyperlink in the insert hyperlink dialog"
+
+  bug fixes:
+    - title: "Fix a regression in the previous release that broke downloading metadata for authors with a double initial such as R. A. Salvatore."
+      tickets: [1294529]
+
+    - title: "Edit book: When generating inline Table of Contents, mark it as such in the guide section of the OPF."
+      tickets: [1287018]
+
+    - title: "E-book viewer: Fix right margin for last page in a chapter sometimes disappearing when changing font size."
+      tickets: [1292822]
+
+    - title: "Edit Book: Fix saving of empty files not working"
+
+    - title: "Edit book: Fix a regression in the previous release that broke saving a copy of the current book on linux and OS X"
+
+    - title: "Edit book: Fix syntax highlighting in HTML files breaks if the closing of a comment or processing instruction is at the start of a new line."
+
+    - title: "Edit book: Fix check book failing in the presence of empty <style/> tags."
+      tickets: [1292841]
+
+    - title: "Catalogs: Fix handling of tristate boolean custom columns when creating EPUB/MOBI catalogs."
+      tickets: [1294983] 
+
+    - title: "Linux build: Workaround for systems that have broken libc implementations that change the behavior of truncate() on file descriptors with O_APPEND set."
+      tickets: [1295366]
+
+  improved recipes:
+    - TIME
+    - Wired Daily Edition
+
+  new recipes:
+    - title: Applefobia 
+      author: koliberek
+
+- version: 1.28.0
+  date: 2014-03-14
+
+  new features:
+    - title: "Edit Book: Add a tool to easily insert hyperlinks (click the insert hyperlink button on the toolbar)"
+
+    - title: "Edit book: Add a tool to easily open a file inside the book for editing by just typing a few characters from the file name. To use it press Ctrl+T in the editor or go to Edit->Quick open a file to edit'"
+
+    - title: "Edit book: Allow disabling the completion popups for the search and replace fields. Right click on the search/replace field to enable/disable the completion popup"
+
+    - title: "E-book viewer: Add an option to control the maximum text height in full screen. Note that it only works if the viewer is in paged mode (which is the default mode)."
+
+    - title: "Show the search expression for the virtual library in a tooltip when hovering over the tab for the virtual library."
+      tickets: [1291691]
+
+    - title: "Book details panel: Show author URL in a tooltip when hovering over author names"
+
+    - title: "Kobo driver: Update to handle updated Kobo firmware"
+
+  bug fixes:
+    - title: "Library backup: Avoid infinite retries if converting metadata to backup OPF for a book fails. Simply fail to backup the metadata for that book."
+      tickets: [1291142]
+
+    - title: "Edit book: Fix file permissions for the edited book being changed on Linux and OS X"
+
+    - title: "Fix text entry cursor becoming invisible when completion popup is opened"
+
+    - title: "Possible fix for crash on OS X Mavericks when adding duplicates"
+
+    - title: "E-book viewer: Fix pressing the Esc key to leave full screen mode not changing the state of the full screen button"
+
+    - title: "When reading metadata from filenames, do not apply the fallback regexp to read metadata if the user specified regexp puts the entire filename into the title. The fallback is only used if the user specified expression does not match the filename at all."
+
+    - title: "Linux binary install script: Fix error on linux systems where the system python has an encoding of None set on stdout. Assume encoding is utf-8 in this case."
+
+    - title: "Content server: Fix (maybe) an error on some windows computers with a non-standard default encoding"
+
+  improved recipes:
+    - Fleshbot
+
 - version: 1.27.0
   date: 2014-03-07
 

manual/conversion.rst

@@ -840,7 +840,7 @@ template::
 This will display the title at the left and the author at the right, in a font
 size smaller than the main text.
 
-Finally, you can also use the current section in templates, as shown below::
+You can also use the current section in templates, as shown below::
 
     <p style="text-align:right">_SECTION_</p>
 
@@ -850,6 +850,12 @@ Outline). If the document has no table of contents then it will be replaced by
 empty text. If a single PDF page has multiple sections, the first section on
 the page will be used.
 
+You can even use javascript inside the header and footer templates, for
+example, the following template will cause page numbers to start at 4 instead
+of 1::
+
+    <p id="pagenum" style="text-align:center;"></p><script>document.getElementById("pagenum").innerHTML = "" + (_PAGENUM_ + 3)</script>
+
 .. note:: When adding headers and footers make sure you set the page top and
     bottom margins to large enough values, under the Page Setup section of the
     conversion dialog.

recipes/american_thinker.recipe

@@ -3,7 +3,15 @@ __copyright__ = '2010, Walt Anthony <workshop.northpole at gmail.com>'
 '''
 www.americanthinker.com
 '''
+import html5lib
 from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.utils.cleantext import clean_xml_chars
+from lxml import etree
+
+def CSSSelect(expr):
+    from cssselect import HTMLTranslator
+    from lxml.etree import XPath
+    return XPath(HTMLTranslator().css_to_xpath(expr))
 
 class AmericanThinker(BasicNewsRecipe):
     title          = u'American Thinker'
@@ -15,9 +23,10 @@ class AmericanThinker(BasicNewsRecipe):
     max_articles_per_feed = 50
     summary_length        = 150
     language              = 'en'
+    ignore_duplicate_articles = {'title', 'url'}
 
     remove_javascript     = True
-    no_stylesheets        = True
+    remove_tags_before = dict(name='h1')
 
     conversion_options = {
                           'comment'   : description
@@ -26,7 +35,14 @@ class AmericanThinker(BasicNewsRecipe):
                         , 'language'  : language
                         , 'linearize_tables' : True
                         }
-    auto_claenup = True
+
+    def preprocess_raw_html(self, raw, url):
+        root = html5lib.parse(
+            clean_xml_chars(raw), treebuilder='lxml',
+            namespaceHTMLElements=False)
+        for x in CSSSelect('.article_body.bottom')(root):
+            x.getparent().remove(x)
+        return etree.tostring(root, encoding=unicode)
 
     feeds = [(u'http://feeds.feedburner.com/americanthinker'),
                      (u'http://feeds.feedburner.com/AmericanThinkerBlog')
@@ -34,4 +50,3 @@ class AmericanThinker(BasicNewsRecipe):
 
     def print_version(self, url):
         return 'http://www.americanthinker.com/assets/3rd_party/printpage/?url=' + url
-        return 'http://www.americanthinker.com/printpage/?url=' + url

recipes/applefobia.recipe

@@ -0,0 +1,22 @@
+# vim:fileencoding=UTF-8
+from __future__ import unicode_literals
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class BasicUserRecipe1395137685(BasicNewsRecipe):
+    title          = u'Applefobia'
+    __author__     = 'koliberek'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    auto_cleanup = True
+    language                  = 'pl'
+    remove_empty_feeds        = True
+    remove_javascript         = True
+    conversion_options        = {
+        'tags'                : u'newsy, Apple, humor',
+        'smarten_punctuation' : True,
+        'authors'             : 'Ogrodnik January',
+        'publisher'           : 'Blogspot.pl'
+        }
+    reverse_article_order     = True
+
+    feeds          = [(u'Aktualne', u'http://applefobia.blogspot.com/feeds/posts/default')]

recipes/atlantic.recipe

@@ -18,13 +18,14 @@ class TheAtlantic(BasicNewsRecipe):
     INDEX = 'http://www.theatlantic.com/magazine/toc/0/'
     language = 'en'
 
-    remove_tags_before = dict(name='div', id='articleHead')
+    keep_only_tags = [{'attrs':{'class':['article', 'articleHead', 'articleText']}}]
-    remove_tags_after  = dict(id='copyright')
+    remove_tags        = [dict(attrs={'class':'footer'})]
-    remove_tags        = [dict(id=['header', 'printAds', 'pageControls'])]
     no_stylesheets = True
 
-    preprocess_regexps = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
+    preprocess_regexps = [
-
+        (re.compile(r'<!--.*?-->', re.DOTALL), lambda m: ''),
+        (re.compile(r'.*<html', re.DOTALL|re.IGNORECASE), lambda m: '<html'),
+    ]
 
     def print_version(self, url):
         return url.replace('/archive/', '/print/')
@@ -40,7 +41,7 @@ class TheAtlantic(BasicNewsRecipe):
         cover = soup.find('img', src=True, attrs={'class':'cover'})
 
         if cover is not None:
-            self.cover_url = re.sub('\s','%20',re.sub('jpg.*','jpg',cover['src']))
+            self.cover_url = 'http:' + cover['src']
             self.log(self.cover_url)
 
         feeds = []
@@ -69,7 +70,7 @@ class TheAtlantic(BasicNewsRecipe):
             if articles:
                 feeds.append((section_title, articles))
 
-        rightContent=soup.find('div', attrs = {'class':'rightContent'})
+        rightContent=soup.find('div', attrs={'class':'rightContent'})
         for module in rightContent.findAll('div', attrs={'class':'module'}):
             section_title = self.tag_to_string(module.find('h2'))
             articles = []
@@ -92,7 +93,6 @@ class TheAtlantic(BasicNewsRecipe):
             if articles:
                 feeds.append((section_title, articles))
 
-
         return feeds
 
     def postprocess_html(self, soup, first):

recipes/courrierinternational.recipe

@@ -1,7 +1,8 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2009, Mathieu Godlewski <mathieu at godlewski.fr>'
+__copyright__ = '''2009, Mathieu Godlewski <mathieu at godlewski.fr>
+2014, Rémi Vanicat <vanicat at debian.org>'''
 '''
 Courrier International
 '''
@@ -19,23 +20,57 @@ class CourrierInternational(BasicNewsRecipe):
     max_articles_per_feed = 50
     no_stylesheets = True
 
+    ignore_duplicate_articles = {'title', 'url'}
+
     html2lrf_options = ['--base-font-size', '10']
 
+    keep_only_tags   = [
+        dict(name='div', attrs={'class':'dessin'}),
+        dict(name='div', attrs={'class':'story-content'}),
+    ]
+    remove_tags      = [
+        dict(name='div', attrs={'class':re.compile('story-share storylinks|pager|event-expand')}),
+        dict(name='li', attrs={'class':'event-partage_outils'}),
+        dict(name='li', attrs={'class':'story-comment-link'}),
+    ]
+
+    needs_subscription = "optional"
+    login_url          = 'http://www.courrierinternational.com/login'
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser(self)
+        if self.username:
+            br.open(self.login_url)
+            br.select_form(nr=1)
+            br['name'] = self.username
+            br['pass'] = self.password
+            br.submit()
+        return br
+
+    def preprocess_html(self, soup):
+        for link in soup.findAll("a",href=re.compile('^/')):
+            link["href"]='http://www.courrierinternational.com' + link["href"]
+        return soup
+
     feeds =  [
         # Some articles requiring subscription fails on download.
         ('A la Une', 'http://www.courrierinternational.com/rss/rss_a_la_une.xml'),
+        ('France', 'http://courrierint.com/rss/rp/14/0/rss.xml'),
+        ('Europe', 'http://courrierint.com/rss/rp/15/0/rss.xml'),
+        ('Amerique', 'http://courrierint.com/rss/rp/16/0/rss.xml'),
+        ('Asie', 'http://courrierint.com/rss/rp/17/0/rss.xml'),
+        ('Afrique', 'http://courrierint.com/rss/rp/18/0/rss.xml'),
+        ('Moyen-Orient', 'http://courrierint.com/rss/rp/19/0/rss.xml'),
+        ('Economie', 'http://courrierint.com/rss/rp/20/0/rss.xml'),
+        ('Multimedia', 'http://courrierint.com/rss/rp/23/0/rss.xml'),
+        ('Sciences', 'http://courrierint.com/rss/rp/22/0/rss.xml'),
+        ('Culture', 'http://courrierint.com/rss/rp/24/0/rss.xml'),
+        ('Insolites', 'http://courrierint.com/rss/rp/26/0/rss.xml'),
+        ('Cartoons', 'http://cs.courrierint.com/rss/all/rss.xml'),
+        ('Environnement', 'http://vt.courrierint.com/rss/all/rss.xml'),
+        ('Cinema', 'http://ca.courrierint.com/rss/all/rss.xml'),
+        ('Sport', 'http://st.courrierint.com/rss/all/rss.xml'),
     ]
 
-    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE|re.DOTALL), i[1]) for i in
-        [
-            #Handle Depeches
-            (r'.*<td [^>]*>([0-9][0-9]/.*</p>)</td>.*', lambda match : '<html><body><table><tr><td>'+match.group(1)+'</td></tr></table></body></html>'),
-            #Handle Articles
-            (r'.*<td [^>]*>(Courrier international.*?)							<td width="10"><img src="/img/espaceur.gif"></td>.*', lambda match : '<html><body><table><tr><td>'+match.group(1)+'</body></html>'),
-        ]
-    ]
-
-
     def print_version(self, url):
-        return re.sub('/[a-zA-Z]+\.asp','/imprimer.asp' ,url)
+        return url + '?page=all'
-

recipes/der_spiegel.recipe

@@ -44,9 +44,9 @@ class DerSpiegel(BasicNewsRecipe):
 
         br = BasicNewsRecipe.get_browser(self)
         if self.username is not None and self.password is not None:
-            br.open(self.PREFIX + '/meinspiegel/login.html')
+            br.open(self.PREFIX + '/meinspiegel/login.html?backUrl=' + self.PREFIX + '/spiegel/print')
             br.select_form(predicate=has_login_name)
-            br['f.loginName'    ] = self.username
+            br['f.loginName'] = self.username
             br['f.password'] = self.password
             br.submit()
         return br
@@ -80,4 +80,4 @@ class DerSpiegel(BasicNewsRecipe):
                 url = self.PREFIX + link['href']
                 articles.append({'title' : title, 'date' : strftime(self.timefmt), 'url' : url})
             feeds.append((section_title,articles))
-        return feeds;
+        return feeds

recipes/fleshbot.recipe

@@ -31,13 +31,12 @@ class Fleshbot(BasicNewsRecipe):
                         , 'language'  : language
                         }
 
-    feeds = [(u'Articles', u'http://www.fleshbot.com/feed')]
+    feeds = [(u'Articles', u'http://fleshbot.com/?feed=rss2')]
 
     remove_tags = [
             {'class': 'feedflare'},
     ]
 
-
     def preprocess_html(self, soup):
         return self.adeify_images(soup)
 

recipes/guardian.recipe

@@ -30,6 +30,8 @@ class Guardian(BasicNewsRecipe):
     max_articles_per_feed       = 100
     remove_javascript           = True
     encoding                    = 'utf-8'
+    compress_news_images = True
+    compress_news_images_auto_size = 8
 
     # List of section titles to ignore
     # For example: ['Sport']
@@ -48,14 +50,14 @@ class Guardian(BasicNewsRecipe):
                         # article history link
                         dict(name='a', attrs={'class':["rollover history-link"]}),
                         # "a version of this article ..." speil
-                        dict(name='div' , attrs = { 'class' : ['section']}),
+                        dict(name='div' , attrs={'class' : ['section']}),
                         # "about this article" js dialog
                         dict(name='div', attrs={'class':["share-top",]}),
                         # author picture
                         dict(name='img', attrs={'class':["contributor-pic-small"]}),
                         # embedded videos/captions
                         dict(name='span',attrs={'class' : ['inline embed embed-media']}),
-                        #dict(name='img'),
+                        # dict(name='img'),
                         ]
     use_embedded_content    = False
 
@@ -104,7 +106,7 @@ class Guardian(BasicNewsRecipe):
 
      # removes number next to rating stars
         items_to_remove = []
-          rating_container = soup.find('div', attrs = {'class': ['rating-container']})
+        rating_container = soup.find('div', attrs={'class': ['rating-container']})
         if rating_container:
             for item in rating_container:
                 if isinstance(item, Tag) and str(item.name) == 'span':
@@ -119,7 +121,7 @@ class Guardian(BasicNewsRecipe):
         # soup = self.index_to_soup("http://www.guardian.co.uk/theobserver")
         soup = self.index_to_soup(self.base_url)
         # find cover pic
-        img = soup.find( 'img',attrs ={'alt':self.cover_pic})
+        img = soup.find('img',attrs={'alt':self.cover_pic})
         if img is not None:
             self.cover_url = img['src']
         # end find cover pic
@@ -149,7 +151,8 @@ class Guardian(BasicNewsRecipe):
                     continue
                 tt = li.find('div', attrs={'class':'trailtext'})
                 if tt is not None:
-                    for da in tt.findAll('a'): da.extract()
+                    for da in tt.findAll('a'):
+                        da.extract()
                     desc = self.tag_to_string(tt).strip()
                 yield {
                         'title': title, 'url':url, 'description':desc,
@@ -161,4 +164,3 @@ class Guardian(BasicNewsRecipe):
         for title, href in self.find_sections():
             feeds.append((title, list(self.find_articles(href))))
         return feeds
-

recipes/tagesspiegel.recipe

@@ -1,20 +1,18 @@
-__license__   = 'GPL v3'
-__copyright__ = '2010 Ingo Paschke <ipaschke@gmail.com>'
-
-'''
-Fetch Tagesspiegel.
-'''
-import string, re
-from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class TagesspiegelRSS(BasicNewsRecipe):
+class TagesspiegelRss(BasicNewsRecipe):
     title          = u'Der Tagesspiegel'
-    __author__ = 'Ingo Paschke'
+    oldest_article = 1
-    language = 'de'
-    oldest_article = 7
     max_articles_per_feed = 100
+    language = 'de'
     publication_type = 'newspaper'
+    auto_cleanup = True
+    no_stylesheets = True
+    remove_stylesheets = True
+    remove_javascript = True
+    remove_empty_feeds = True
+    encoding = 'utf-8'
+    use_embedded_content = False
 
     extra_css = '''
                 .hcf-overline{color:#990000; font-family:Arial,Helvetica,sans-serif;font-size:xx-small;display:block}
@@ -30,69 +28,34 @@ class TagesspiegelRSS(BasicNewsRecipe):
                 .hcf-smart-box{font-family: Arial, Helvetica, sans-serif; font-size: xx-small; margin: 0px 15px 8px 0px; width: 300px;}
                 '''
 
-    no_stylesheets = True
-    no_javascript = True
-    remove_empty_feeds = True
-    encoding = 'utf-8'
     remove_tags = [{'class':'hcf-header'}, {'class':'hcf-atlas'}, {'class':'hcf-colon'}, {'class':'hcf-date hcf-separate'}]
 
+    feeds          = [
+        (u'Politik', u'http://www.tagesspiegel.de/contentexport/feed/politik'),
+        (u'Meinung', u'http://www.tagesspiegel.de/contentexport/feed/meinung'),
+        (u'Berlin', u'http://www.tagesspiegel.de/contentexport/feed/berlin'),
+        (u'Wirtschaft', u'http://www.tagesspiegel.de/contentexport/feed/wirtschaft'),
+        (u'Sport', u'http://www.tagesspiegel.de/contentexport/feed/sport'),
+        (u'Kultur', u'http://www.tagesspiegel.de/contentexport/feed/kultur'),
+        (u'Weltspiegel', u'http://www.tagesspiegel.de/contentexport/feed/weltspiegel'),
+        (u'Medien', u'http://www.tagesspiegel.de/contentexport/feed/medien'),
+        (u'Wissen', u'http://www.tagesspiegel.de/contentexport/feed/wissen')
+    ]
+
     def print_version(self, url):
-        url = url.split('/')
+        # print url
+        u = url.find('0L0Stagesspiegel0Bde')
+        u = 'http://www.tagesspiegel.de' + url[u + 20:]
+        u = u.replace('0C', '/')
+        u = u.replace('0E', '-')
+        u = u.replace('A', '')
+        u = u.replace('0B', '.')
+        u = u.replace('.html/story01.htm', '.html')
+        url = u.split('/')
         url[-1] = 'v_print,%s?p='%url[-1]
-        return '/'.join(url)
+        u = '/'.join(url)
+        # print u
+        return u
 
     def get_masthead_url(self):
         return 'http://www.tagesspiegel.de/images/tsp_logo/3114/6.png'
-
-    def parse_index(self):
-        soup = self.index_to_soup('http://www.tagesspiegel.de/zeitung/')
-
-        def feed_title(div):
-            return ''.join(div.findAll(text=True, recursive=False)).strip() if div is not None else None
-
-        articles = {}
-        links = set()
-        key = None
-        ans = []
-        maincol = soup.find('div', attrs={'class':re.compile('hcf-main-col')})
-
-        for div in maincol.findAll(True, attrs={'class':['hcf-teaser', 'hcf-header', 'story headline', 'hcf-teaser hcf-last']}):
-
-             if div['class'] == 'hcf-header':
-                 try:
-                     key = string.capwords(feed_title(div.em))
-                     articles[key] = []
-                     ans.append(key)
-                 except:
-                     continue
-
-             elif div['class'] in ['hcf-teaser', 'hcf-teaser hcf-last'] and getattr(div.contents[0],'name','') == 'h2':
-                 a = div.find('a', href=True)
-                 if not a:
-                     continue
-                 url = 'http://www.tagesspiegel.de' + a['href']
-
-                 # check for duplicates
-                 if url in links:
-                     continue	
-                 links.add(url)					 
-
-                 title = self.tag_to_string(a, use_alt=True).strip()
-                 description = ''
-                 pubdate = strftime('%a, %d %b')
-                 summary = div.find('p', attrs={'class':'hcf-teaser'})
-                 if summary:
-                     description = self.tag_to_string(summary, use_alt=False)
-
-                 feed = key if key is not None else 'Uncategorized'
-                 if not articles.has_key(feed):
-                     articles[feed] = []
-                 if not 'podcasts' in url:
-                     articles[feed].append(
-                               dict(title=title, url=url, date=pubdate,
-                                    description=re.sub('mehr$', '', description),
-                                    content=''))
-
-        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
-
-        return ans

recipes/time_magazine.recipe

@@ -13,17 +13,17 @@ from calibre.web.feeds.jsnews import JavascriptRecipe
 from lxml import html
 
 def wait_for_load(browser):
-    # This element is present in the black login bar at the top
+    # This element is present next to the main TIME logo in the left hand side nav bar
-    browser.wait_for_element('#site-header p.constrain', timeout=180)
+    browser.wait_for_element('.signedin-wrap a[href]', timeout=180)
 
 # Keep the login method as standalone, so it can be easily tested
 def do_login(browser, username, password):
     from calibre.web.jsbrowser.browser import Timeout
-    browser.visit('http://www.time.com/time/magazine')
+    browser.visit('http://time.com/magazine')
-    form = browser.select_form('#magazine-signup')
+    form = browser.select_form('#sign-in-form')
     form['username'] = username
     form['password'] = password
-    browser.submit('#paid-wall-submit')
+    browser.submit('#Sign_In')
     try:
         wait_for_load(browser)
     except Timeout:
@@ -40,100 +40,57 @@ class Time(JavascriptRecipe):
 
     no_stylesheets        = True
     remove_javascript     = True
-    keep_only_tags = ['article.post']
+    keep_only_tags = ['.article-viewport .full-article']
-    remove_tags = ['meta', '.entry-sharing', '.entry-footer', '.wp-paginate',
+    remove_tags = ['.read-more-list', '.read-more-inline', '.article-footer', '.subscribe', '.tooltip', '#first-visit']
-                   '.post-rail', '.entry-comments', '.entry-tools',
-                   '#paid-wall-cm-ad']
-
-    recursions = 1
-    links_from_selectors = ['.wp-paginate a.page[href]']
-
-    extra_css = '.entry-date { padding-left: 2ex }'
 
     def do_login(self, browser, username, password):
         do_login(browser, username, password)
 
-    def get_publication_data(self, browser):
+    def get_time_cover(self, browser):
-        selector = 'section.sec-mag-showcase ul.ul-mag-showcase img[src]'
+        selector = '#rail-articles img.magazine-thumb'
         cover = browser.css_select(selector)
         # URL for large cover
-        cover_url = unicode(cover.evaluateJavaScript('this.src').toString()).replace('_400.', '_600.')
+        cover_url = unicode(cover.evaluateJavaScript('this.src').toString()).partition('?')[0] + '?w=814'
-        raw = browser.html
+        return browser.get_resource(cover_url)
-        ans = {'cover': browser.get_resource(cover_url)}
+
+    def get_publication_data(self, browser):
         # We are already at the magazine page thanks to the do_login() method
+        ans = {}
+        raw = browser.html
         root = html.fromstring(raw)
 
-        dates = ''.join(root.xpath('//time[@class="updated"]/text()'))
+        dates = ''.join(root.xpath('//*[@class="rail-article-magazine-issue"]/date/text()'))
         if dates:
             self.timefmt = ' [%s]'%dates
 
-        feeds = []
+        parent = root.xpath('//section[@id="rail-articles"]')[0]
-        parent = root.xpath('//div[@class="content-main-aside"]')[0]
+        articles = []
-        for sec in parent.xpath(
+        for h3 in parent.xpath(
-                'descendant::section[contains(@class, "sec-mag-section")]'):
+                'descendant::h3[contains(@class, "rail-article-title")]'):
-            h3 = sec.xpath('./h3')
+            title = html.tostring(h3[0], encoding=unicode, method='text').strip()
-            if h3:
+            a = h3.xpath('descendant::a[@href]')[0]
-                section = html.tostring(h3[0], encoding=unicode,
+            url = a.get('href')
-                        method='text').strip().capitalize()
+            h2 = h3.xpath('following-sibling::h2[@class="rail-article-excerpt"]')
-                self.log('Found section', section)
+            desc = ''
-                articles = list(self.find_articles(sec))
+            if h2:
-                if articles:
+                desc = html.tostring(h2[0], encoding=unicode, method='text').strip()
-                    feeds.append((section, articles))
+            self.log('\nFound article:', title)
+            self.log('\t' + desc)
+            articles.append({'title':title, 'url':url, 'date':'', 'description':desc})
 
-        ans['index'] = feeds
+        ans['index'] = [('Articles', articles)]
+        ans['cover'] = self.get_time_cover(browser)
         return ans
 
-    def find_articles(self, sec):
+    def load_complete(self, browser, url, rl):
-        for article in sec.xpath('./article'):
+        browser.wait_for_element('footer.article-footer')
-            h2 = article.xpath('./*[@class="entry-title"]')
-            if not h2:
-                continue
-            a = h2[0].xpath('./a[@href]')
-            if not a:
-                continue
-            title = html.tostring(a[0], encoding=unicode,
-                        method='text').strip()
-            if not title:
-                continue
-            url = a[0].get('href')
-            if url.startswith('/'):
-                url = 'http://www.time.com'+url
-            desc = ''
-            p = article.xpath('./*[@class="entry-content"]')
-            if p:
-                desc = html.tostring(p[0], encoding=unicode,
-                        method='text')
-            self.log('\t', title, ':\n\t\t', url)
-            yield {
-                    'title' : title,
-                    'url'   : url,
-                    'date'  : '',
-                    'description' : desc
-                    }
-
-    def load_complete(self, browser, url, recursion_level):
-        # This is needed as without it, subscriber content is blank. time.com
-        # appears to be using some crazy iframe+js callback for loading content
-        wait_for_load(browser)
         return True
 
     def postprocess_html(self, article, root, url, recursion_level):
-        # Remove the header and page n of m messages from pages after the first
+        # get rid of the first visit div which for some reason remove_tags is
-        # page
+        # not removing
-        if recursion_level > 0:
+        for div in root.xpath('//*[@id="first-visit"]'):
-            for h in root.xpath('//header[@class="entry-header"]|//span[@class="page"]'):
+            div.getparent().remove(div)
-                h.getparent().remove(h)
-        # Unfloat the article images and also remove them from pages after the
-        # first page as they are repeated on every page.
-        for fig in root.xpath('//figure'):
-            parent = fig.getparent()
-            if recursion_level > 0:
-                parent.remove(fig)
-            else:
-                idx = parent.index(fig)
-                for img in reversed(fig.xpath('descendant::img')):
-                    parent.insert(idx, img)
-                parent.remove(fig)
         return root
 
 if __name__ == '__main__':

recipes/wired_daily.recipe

@@ -2,10 +2,8 @@
 __license__   = 'GPL v3'
 __docformat__ = 'restructuredtext en'
 
-import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
-from calibre.ebooks.chardet import xml_to_unicode
 
 class Wired_Daily(BasicNewsRecipe):
 
@@ -14,22 +12,13 @@ class Wired_Daily(BasicNewsRecipe):
     description = 'Technology news'
     timefmt  = ' [%Y%b%d  %H%M]'
     language = 'en'
-
+    use_embedded_content = False
     no_stylesheets = True
 
-    preprocess_regexps = [(re.compile(r'<head.*</head>', re.DOTALL), lambda m:
+    keep_only_tags = [  # dict(name= 'div', id ='liveblog-hdr'),
-        '<head></head>')]
+                      dict(name='div', attrs={'class': 'post'})]
-
-    remove_tags_before = dict(name='div', id='content')
-    remove_tags = [dict(id=['header', 'commenting_module', 'post_nav',
-        'social_tools', 'sidebar', 'footer', 'social_wishlist', 'pgwidget',
-        'outerWrapper', 'inf_widget']),
-        {'class':['entryActions', 'advertisement', 'entryTags']},
-        dict(name=['noscript', 'script']),
-        dict(name='h4', attrs={'class':re.compile(r'rat\d+')}),
-        {'class':lambda x: x and x.startswith('contentjump')},
-        dict(name='li', attrs={'class':['entryCategories', 'entryEdit']})]
 
+    remove_tags = [dict(name='div', attrs={'class': 'social-top'})]
 
     feeds = [
         ('Top News', 'http://feeds.wired.com/wired/index'),
@@ -49,11 +38,8 @@ class Wired_Daily(BasicNewsRecipe):
         ('Science', 'http://www.wired.com/wiredscience/feed/'),
         ]
 
-    def populate_article_metadata(self, article, soup, first):
+    def preprocess_html(self, soup):
-        if article.text_summary:
+        for img in soup.findAll('img', attrs={'data-lazy-src':True}):
-            article.text_summary = xml_to_unicode(article.text_summary,
+            img['src'] = img['data-lazy-src']
-                    resolve_entities=True)[0]
+        return soup
-
-    def print_version(self, url):
-        return url + '/all/1'
 

resources/calibre-ebook-root-CA.crt

@@ -1,32 +1,34 @@
 -----BEGIN CERTIFICATE-----
-MIIFlzCCA3+gAwIBAgIJAI67A/kD1DLtMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV
+MIIFzjCCA7agAwIBAgIJAPE9riMS7RUZMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV
 BAYTAklOMRQwEgYDVQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAw
-DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAeFw0x
+DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAgFw0x
-NDAyMjMwNDAzNDFaFw0xNDAzMjUwNDAzNDFaMGIxCzAJBgNVBAYTAklOMRQwEgYD
+NDAzMjUxMDU2MThaGA8yMTE0MDMwMTEwNTYxOFowYjELMAkGA1UEBhMCSU4xFDAS
-VQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAwDgYDVQQKDAdjYWxp
+BgNVBAgMC01haGFyYXNodHJhMQ8wDQYDVQQHDAZNdW1iYWkxEDAOBgNVBAoMB2Nh
-YnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTCCAiIwDQYJKoZIhvcNAQEB
+bGlicmUxGjAYBgNVBAMMEWNhbGlicmUtZWJvb2suY29tMIICIjANBgkqhkiG9w0B
-BQADggIPADCCAgoCggIBALZW3gMUCsloaMcGhqjIeZLUYarC0ers47qlpgfjJnwt
+AQEFAAOCAg8AMIICCgKCAgEAtlbeAxQKyWhoxwaGqMh5ktRhqsLR6uzjuqWmB+Mm
-DYuOZjkqNkf7rBUE2XrK2FKKNsgYTDefArC3rmmkH7D3g7LO8yfY19L/xmFEt7zO
+fC0Ni45mOSo2R/usFQTZesrYUoo2yBhMN58CsLeuaaQfsPeDss7zJ9jX0v/GYUS3
-6hOea7kVrtINdTabli2ZKr3MOYFYt2SWMf8qkxBpQgxsY11bPYhIPi++QXJvcvO6
+vM7qE55ruRWu0g11NpuWLZkqvcw5gVi3ZJYx/yqTEGlCDGxjXVs9iEg+L75Bcm9y
-JW3GQOh/wm0eZT9f7V3Msm9UwSDbk3IONPEp4nmPx6ZwNa9zUAfTMH0nHV9PB0wd
+87olbcZA6H/CbR5lP1/tXcyyb1TBINuTcg408SnieY/HpnA1r3NQB9MwfScdX08H
-AXPHtKs/q9QTYt8GWXKzaalocOl/UJB4oBmgzaaZlqnNUOZ8cZNqwttRkYOep6er
+TB0Bc8e0qz+r1BNi3wZZcrNpqWhw6X9QkHigGaDNppmWqc1Q5nxxk2rC21GRg56n
-dxDUDHLRNykyX0fE8DN9zf3X3IKGw2f2U56IKnRUMnBToL0+JiGbF3bCb+rJsoZZ
+p6t3ENQMctE3KTJfR8TwM33N/dfcgobDZ/ZTnogqdFQycFOgvT4mIZsXdsJv6smy
-FKsntj1fF3EzSa/sEcyDf/rtt4wvgmk9FNAOew/D1GVYU/mbIV4wfdSqPISxNUpi
+hlkUqye2PV8XcTNJr+wRzIN/+u23jC+CaT0U0A57D8PUZVhT+ZshXjB91Ko8hLE1
-ZHb9m8RVeNm7HpoUsWVgrbHNjb/Pw7PllVdNMXwA8pvi6JMxKqn3Cvb5JDBsxYe8
+SmJkdv2bxFV42bsemhSxZWCtsc2Nv8/Ds+WVV00xfADym+LokzEqqfcK9vkkMGzF
-M3e2KjzqzBjgnvbx9QqC91TubKz1ftDKdX4yBoJuUiIZJckX2niIxXsqA0QOnvBF
+h7wzd7YqPOrMGOCe9vH1CoL3VO5srPV+0Mp1fjIGgm5SIhklyRfaeIjFeyoDRA6e
-6yN8TrK5F1zCQ74Z3RCTmGKqZWPuJC4VtF3k2Yyuwpg+fcUbRWFmld3XDJWlm1cb
+8EXrI3xOsrkXXMJDvhndEJOYYqplY+4kLhW0XeTZjK7CmD59xRtFYWaV3dcMlaWb
-mO3YLIju4lM7WGNE6OWQxMXB3puzxD1E8hYovS4W3EiXlw2qjxTMYofl9Iqir54v
+VxuY7dgsiO7iUztYY0To5ZDExcHem7PEPUTyFii9LhbcSJeXDaqPFMxih+X0iqKv
-AgMBAAGjUDBOMB0GA1UdDgQWBBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAfBgNVHSME
+ni8CAwEAAaOBhDCBgTAxBgNVHREEKjAoghFjYWxpYnJlLWVib29rLmNvbYITKi5j
-GDAWgBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3
+YWxpYnJlLWVib29rLmNvbTAdBgNVHQ4EFgQURWqz5EOg5K1OrSKpleR+louVxsQw
-DQEBBQUAA4ICAQBAlBhF+greu0vYEDzz04HQjgfamxWQ4nXete8++9et1mcRw16i
+HwYDVR0jBBgwFoAURWqz5EOg5K1OrSKpleR+louVxsQwDAYDVR0TBAUwAwEB/zAN
-RbEz/1ZeELz9KMwMpooVPIaYAWgqe+UNWuzHt0+jrH30NBcBv407G8eR/FWOU/cx
+BgkqhkiG9w0BAQUFAAOCAgEANxijK3JQNZnrDYv7E5Ny17EtxV6ADggs8BIFLHrp
-y/YMk3nXsAARoOcsFN1YSS1dNL1osezfsRStET8/bOEqpWD0yvt8wRWwh1hOCPVD
+tRISYw8HpFIrIF/MDbHgYGp/xkefGKGEHeS7rUPYwdAbKM0sfoxKXm5e8GGe9L5K
-OpWTZx7+dZcK1Zh64Rm5mPYzbhWYxGGqNuZGFCuR9yI2bHHsFI69LryUKNf1cJ/N
+pdG+ig1Ptm+Pae2Rcdj9RHKGmpAiKIF8a15l/Yj3jDVk06kx+lnT5fOePGhZBeuj
-dfvHt4GDxfF5ie4PWNgTp52wuI3YxNpsHgz9SmSEey6uVlA13vTO1QFX8Ymbyn6K
+duBZ2vP39rFfcBtTvFmoQRwfoa46fZEoWoXb3YwzBqIhBg9m80R+E79/HsRPwA4L
-FRhr2LHY4iBdY+Gw47WnAqdo7uXpyM3wT6jI4gn7oENvCSUyM/JMSQqE1Etw0LBr
+pOvcFTr28jNp1OadgZ92sY9EYabes23amebz/P6IOjutqssIdrPSKqM9aphlGLXE
-NIlC/RxN5wjcDvVCL/uS3PL6IW7R0wxrCQwBU3f5wMOnDM/R4EWJdS96zyb7Xnh3
+7YDxS9nSfX165Aa8NIWO95ivdbZplisnQ3rQM4pIdk7Z8FPhHftMdhekDREMxYKX
-PQGoj6/vllymI7tuwRhEuvFknRRihu3vilHgtGczVXTG73nFJftLzvN/OhqSSQG/
+KXepi5tLyVnhETj+ifYBwqxZ024rlnpnHUWgjxRz5atKTAsbAgcxHOYTKMZoRAod
-3c2JDX+vAy5jwPT/M3nPkrs68M4P77da1/BDZ0/KgJb/JzYZyNpq1nhWo3nMn+Sx
+BK7lvjZ7+C/cqUc2c9FSG/HxkrfMpJHJlzMsanTBJ1+MeUybeBtp5E7gdNALbfh/
-jq7y+h6ry8Omnlw7a/7CnNgvkLfP/uTfllL4erETFntHNh6LqCvpPNOqrvAP5keB
+BJ4eWw7X7q2oKape+7+OMX7aKAIysM7d2iVRuBofLBxOqzY6mzP8+Ro8zIgwFUeh
-EB8yoJraypfuiNELOw1zSRksMxe2ac4b/dhDNStBTPC0egfRSm3FA0XoOQ==
+r6pbEa8P2DXnuZ+PtcMiClYKuSLlf6xRRDMnHCxvsu1zA/Ga3vZ6g0bd487DIsGP
+tXHCYXttMGNxZDNVKS6rkrY2sT5xnJwvHwWmiooUZmSUFUdpqsvV5r9v89NMQ87L
+gNA=
 -----END CERTIFICATE-----

resources/default_tweaks.py

@@ -147,7 +147,7 @@ sort_columns_at_startup = None
 #  d     the day as number without a leading zero (1 to 31)
 #  dd    the day as number with a leading zero (01 to 31)
 #  ddd   the abbreviated localized day name (e.g. 'Mon' to 'Sun').
-#  dddd  the long localized day name (e.g. 'Monday' to 'Qt::Sunday').
+#  dddd  the long localized day name (e.g. 'Monday' to 'Sunday').
 #  M     the month as number without a leading zero (1-12)
 #  MM    the month as number with a leading zero (01-12)
 #  MMM   the abbreviated localized month name (e.g. 'Jan' to 'Dec').
@@ -444,7 +444,7 @@ public_smtp_relay_delay = 301
 # All covers in the calibre library will be resized, preserving aspect ratio,
 # to fit within this size. This is to prevent slowdowns caused by extremely
 # large covers
-maximum_cover_size = (1450, 2000)
+maximum_cover_size = (1650, 2200)
 
 #: Where to send downloaded news
 # When automatically sending downloaded news to a connected device, calibre

session.vim

@@ -17,9 +17,9 @@ let g:syntastic_cpp_include_dirs = [
             \]
 let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs
 
-set wildignore+=resources/viewer/mathjax/**
+set wildignore+=resources/viewer/mathjax/*
-set wildignore+=build/**
+set wildignore+=build/*
-set wildignore+=dist/**
+set wildignore+=dist/*
 
 fun! CalibreLog()
     " Setup buffers to edit the calibre changelog and version info prior to

setup/extensions.py

@@ -179,12 +179,12 @@ extensions = [
     ),
 
     Extension('matcher',
-        ['calibre/gui2/tweak_book/matcher.c'],
+        ['calibre/utils/matcher.c'],
         headers=['calibre/utils/icu_calibre_utils.h'],
         libraries=icu_libs,
         lib_dirs=icu_lib_dirs,
         cflags=icu_cflags,
-        inc_dirs=icu_inc_dirs  + ['calibre/utils']
+        inc_dirs=icu_inc_dirs
     ),
 
     Extension('podofo',
@@ -303,9 +303,10 @@ if islinux or isosx:
 if isunix:
     cc = os.environ.get('CC', 'gcc')
     cxx = os.environ.get('CXX', 'g++')
+    debug = ''
+    # debug = '-ggdb'
     cflags = os.environ.get('OVERRIDE_CFLAGS',
-        # '-Wall -DNDEBUG -ggdb -fno-strict-aliasing -pipe')
+        '-Wall -DNDEBUG %s -fno-strict-aliasing -pipe' % debug)
-        '-Wall -DNDEBUG -fno-strict-aliasing -pipe')
     cflags = shlex.split(cflags) + ['-fPIC']
     ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall')
     ldflags = shlex.split(ldflags)

setup/installer/__init__.py

@@ -14,7 +14,7 @@ from setup.build_environment import HOST, PROJECT
 BASE_RSYNC = ['rsync', '-avz', '--delete', '--force']
 EXCLUDES = []
 for x in [
-    'src/calibre/plugins', 'src/calibre/manual', 'src/calibre/trac',
+    'src/calibre/plugins', 'manual',
     '.bzr', '.git', '.build', '.svn', 'build', 'dist', 'imgsrc', '*.pyc', '*.pyo', '*.swp',
     '*.swo', 'format_docs']:
     EXCLUDES.extend(['--exclude', x])
@@ -82,6 +82,7 @@ class Push(Command):
                 r'kovid@win7:/cygdrive/c/Users/kovid/calibre':'Windows 7',
                 'kovid@win7-x64:calibre-src':'win7-x64',
                 'kovid@tiny:calibre':None,
+                'kovid@getafix:calibre-src':None,
                 }.iteritems():
                 threads[vmname or host] = thread = Thread(target=push, args=(host, vmname, available))
                 thread.start()

setup/installer/linux/freeze2.py

@@ -279,6 +279,12 @@ class LinuxFreeze(Command):
         modules['console'].append('calibre.linux')
         basenames['console'].append('calibre_postinstall')
         functions['console'].append('main')
+        c_launcher = '/tmp/calibre-c-launcher'
+        lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c')
+        cmd = ['gcc', '-O2', '-DMAGICK_BASE="%s"' % self.magick_base, '-o', c_launcher, lsrc, ]
+        self.info('Compiling launcher')
+        self.run_builder(cmd, verbose=False)
+
         for typ in ('console', 'gui', ):
             self.info('Processing %s launchers'%typ)
             for mod, bname, func in zip(modules[typ], basenames[typ],
@@ -288,20 +294,6 @@ class LinuxFreeze(Command):
                 xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname,
                     '-DFUNCTION="%s"'%func]
 
-                launcher = textwrap.dedent('''\
-                #!/bin/sh
-                path=`readlink -f $0`
-                base=`dirname $path`
-                lib=$base/lib
-                export QT_ACCESSIBILITY=0 # qt-at-spi causes crashes and performance issues in various distros, so disable it
-                export LD_LIBRARY_PATH=$lib:$LD_LIBRARY_PATH
-                export MAGICK_HOME=$base
-                export MAGICK_CONFIGURE_PATH=$lib/{1}/config
-                export MAGICK_CODER_MODULE_PATH=$lib/{1}/modules-Q16/coders
-                export MAGICK_CODER_FILTER_PATH=$lib/{1}/modules-Q16/filters
-                exec $base/bin/{0} "$@"
-                ''')
-
                 dest = self.j(self.obj_dir, bname+'.o')
                 if self.newer(dest, [src, __file__]+headers):
                     self.info('Compiling', bname)
@@ -309,8 +301,7 @@ class LinuxFreeze(Command):
                     self.run_builder(cmd, verbose=False)
                 exe = self.j(self.bin_dir, bname)
                 sh = self.j(self.base, bname)
-                with open(sh, 'wb') as f:
+                shutil.copy2(c_launcher, sh)
-                    f.write(launcher.format(bname, self.magick_base))
                 os.chmod(sh,
                     stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
 

setup/installer/linux/launcher.c

@@ -0,0 +1,74 @@
+/*
+ * launcher.c
+ * Copyright (C) 2014 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <libgen.h>
+#include <stdlib.h>
+
+#define PATHLEN 1023
+
+int main(int argc, char **argv) {
+    static char buf[PATHLEN+1] = {0}, lib[PATHLEN+1] = {0}, base[PATHLEN+1] = {0}, exe[PATHLEN+1] = {0}, *ldp = NULL;
+
+    if (readlink("/proc/self/exe", buf, PATHLEN) == -1) {
+        fprintf(stderr, "Failed to read path of executable with error: %s\n", strerror(errno));
+        return 1;
+    }
+    strncpy(lib, buf, PATHLEN);
+    strncpy(base, dirname(lib), PATHLEN);
+    snprintf(exe, PATHLEN, "%s/bin/%s", base, basename(buf));
+    memset(lib, 0, PATHLEN);
+    snprintf(lib, PATHLEN, "%s/lib", base);
+
+    /* qt-at-spi causes crashes and performance issues in various distros, so disable it */
+    if (setenv("QT_ACCESSIBILITY", "0", 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+
+    if (setenv("MAGICK_HOME", base, 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+    memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/config", lib, MAGICK_BASE);
+    if (setenv("MAGICK_CONFIGURE_PATH", buf, 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+    memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/modules-Q16/coders", lib, MAGICK_BASE);
+    if (setenv("MAGICK_CODER_MODULE_PATH", buf, 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+    memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/modules-Q16/filters", lib, MAGICK_BASE);
+    if (setenv("MAGICK_CODER_FILTER_PATH", buf, 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+
+    memset(buf, 0, PATHLEN);
+    ldp = getenv("LD_LIBRARY_PATH");
+    if (ldp == NULL) strncpy(buf, lib, PATHLEN);
+    else snprintf(buf, PATHLEN, "%s:%s", lib, ldp);
+    if (setenv("LD_LIBRARY_PATH", buf, 1) != 0) {
+        fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno));
+        return 1;
+    }
+
+    argv[0] = exe;
+    if (execv(exe, argv) == -1) {
+        fprintf(stderr, "Failed to execute binary: %s with error: %s\n", exe, strerror(errno));
+        return 1;
+    }
+
+    return 0;
+}
+
+
+

setup/installer/windows/notes.rst

@@ -60,6 +60,9 @@ to login as the normal user account with ssh. To do this, follow these steps:
       http://pcsupport.about.com/od/windows7/ht/auto-logon-windows-7.htm or
       http://pcsupport.about.com/od/windowsxp/ht/auto-logon-xp.htm to allow the
       machine to bootup without having to enter the password
+
+    * The following steps must all be run in an administrator cygwin shell
+
     * First clean out any existing cygwin ssh setup with::
         net stop sshd
         cygrunsrv -R sshd
@@ -70,7 +73,7 @@ to login as the normal user account with ssh. To do this, follow these steps:
         mkpasswd -cl > /etc/passwd
         mkgroup --local > /etc/group
     * Assign the necessary rights to the normal user account (administrator
-      command prompt needed)::
+      cygwin command prompt needed - editrights is available in \cygwin\bin)::
         editrights.exe -a SeAssignPrimaryTokenPrivilege -u kovid
         editrights.exe -a SeCreateTokenPrivilege -u kovid
         editrights.exe -a SeTcbPrivilege -u kovid

setup/linux-installer.py

@@ -19,6 +19,7 @@ py3 = sys.version_info[0] > 2
 enc = getattr(sys.stdout, 'encoding', 'UTF-8') or 'utf-8'
 calibre_version = signature = None
 urllib = __import__('urllib.request' if py3 else 'urllib', fromlist=1)
+
 if py3:
     unicode = str
     raw_input = input
@@ -448,9 +449,15 @@ def match_hostname(cert, hostname):
             "doesn't match either of %s"
             % (hostname, ', '.join(map(repr, dnsnames))))
     elif len(dnsnames) == 1:
-        # python 2.6 does not read subjectAltName, so we do the best we can
+        # python 2.7.2 does not read subject alt names thanks to this
-        if sys.version_info[:2] == (2, 6):
+        # bug: http://bugs.python.org/issue13034
-            if dnsnames[0] == 'calibre-ebook.com':
+        # And the utter lunacy that is the linux landscape could have
+        # any old version of python whatsoever with or without a hot fix for
+        # this bug. Not to mention that python 2.6 may or may not
+        # read alt names depending on its patchlevel. So we just bail on full
+        # verification if the python version is less than 2.7.3.
+        # Linux distros are one enormous, honking disaster.
+        if sys.version_info[:3] < (2, 7, 3) and dnsnames[0] == 'calibre-ebook.com':
             return
         raise CertificateError("hostname %r "
             "doesn't match %r"
@@ -494,36 +501,38 @@ else:
 
 CACERT = b'''\
 -----BEGIN CERTIFICATE-----
-MIIFlzCCA3+gAwIBAgIJAI67A/kD1DLtMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV
+MIIFzjCCA7agAwIBAgIJAPE9riMS7RUZMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV
 BAYTAklOMRQwEgYDVQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAw
-DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAeFw0x
+DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAgFw0x
-NDAyMjMwNDAzNDFaFw0xNDAzMjUwNDAzNDFaMGIxCzAJBgNVBAYTAklOMRQwEgYD
+NDAzMjUxMDU2MThaGA8yMTE0MDMwMTEwNTYxOFowYjELMAkGA1UEBhMCSU4xFDAS
-VQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAwDgYDVQQKDAdjYWxp
+BgNVBAgMC01haGFyYXNodHJhMQ8wDQYDVQQHDAZNdW1iYWkxEDAOBgNVBAoMB2Nh
-YnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTCCAiIwDQYJKoZIhvcNAQEB
+bGlicmUxGjAYBgNVBAMMEWNhbGlicmUtZWJvb2suY29tMIICIjANBgkqhkiG9w0B
-BQADggIPADCCAgoCggIBALZW3gMUCsloaMcGhqjIeZLUYarC0ers47qlpgfjJnwt
+AQEFAAOCAg8AMIICCgKCAgEAtlbeAxQKyWhoxwaGqMh5ktRhqsLR6uzjuqWmB+Mm
-DYuOZjkqNkf7rBUE2XrK2FKKNsgYTDefArC3rmmkH7D3g7LO8yfY19L/xmFEt7zO
+fC0Ni45mOSo2R/usFQTZesrYUoo2yBhMN58CsLeuaaQfsPeDss7zJ9jX0v/GYUS3
-6hOea7kVrtINdTabli2ZKr3MOYFYt2SWMf8qkxBpQgxsY11bPYhIPi++QXJvcvO6
+vM7qE55ruRWu0g11NpuWLZkqvcw5gVi3ZJYx/yqTEGlCDGxjXVs9iEg+L75Bcm9y
-JW3GQOh/wm0eZT9f7V3Msm9UwSDbk3IONPEp4nmPx6ZwNa9zUAfTMH0nHV9PB0wd
+87olbcZA6H/CbR5lP1/tXcyyb1TBINuTcg408SnieY/HpnA1r3NQB9MwfScdX08H
-AXPHtKs/q9QTYt8GWXKzaalocOl/UJB4oBmgzaaZlqnNUOZ8cZNqwttRkYOep6er
+TB0Bc8e0qz+r1BNi3wZZcrNpqWhw6X9QkHigGaDNppmWqc1Q5nxxk2rC21GRg56n
-dxDUDHLRNykyX0fE8DN9zf3X3IKGw2f2U56IKnRUMnBToL0+JiGbF3bCb+rJsoZZ
+p6t3ENQMctE3KTJfR8TwM33N/dfcgobDZ/ZTnogqdFQycFOgvT4mIZsXdsJv6smy
-FKsntj1fF3EzSa/sEcyDf/rtt4wvgmk9FNAOew/D1GVYU/mbIV4wfdSqPISxNUpi
+hlkUqye2PV8XcTNJr+wRzIN/+u23jC+CaT0U0A57D8PUZVhT+ZshXjB91Ko8hLE1
-ZHb9m8RVeNm7HpoUsWVgrbHNjb/Pw7PllVdNMXwA8pvi6JMxKqn3Cvb5JDBsxYe8
+SmJkdv2bxFV42bsemhSxZWCtsc2Nv8/Ds+WVV00xfADym+LokzEqqfcK9vkkMGzF
-M3e2KjzqzBjgnvbx9QqC91TubKz1ftDKdX4yBoJuUiIZJckX2niIxXsqA0QOnvBF
+h7wzd7YqPOrMGOCe9vH1CoL3VO5srPV+0Mp1fjIGgm5SIhklyRfaeIjFeyoDRA6e
-6yN8TrK5F1zCQ74Z3RCTmGKqZWPuJC4VtF3k2Yyuwpg+fcUbRWFmld3XDJWlm1cb
+8EXrI3xOsrkXXMJDvhndEJOYYqplY+4kLhW0XeTZjK7CmD59xRtFYWaV3dcMlaWb
-mO3YLIju4lM7WGNE6OWQxMXB3puzxD1E8hYovS4W3EiXlw2qjxTMYofl9Iqir54v
+VxuY7dgsiO7iUztYY0To5ZDExcHem7PEPUTyFii9LhbcSJeXDaqPFMxih+X0iqKv
-AgMBAAGjUDBOMB0GA1UdDgQWBBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAfBgNVHSME
+ni8CAwEAAaOBhDCBgTAxBgNVHREEKjAoghFjYWxpYnJlLWVib29rLmNvbYITKi5j
-GDAWgBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3
+YWxpYnJlLWVib29rLmNvbTAdBgNVHQ4EFgQURWqz5EOg5K1OrSKpleR+louVxsQw
-DQEBBQUAA4ICAQBAlBhF+greu0vYEDzz04HQjgfamxWQ4nXete8++9et1mcRw16i
+HwYDVR0jBBgwFoAURWqz5EOg5K1OrSKpleR+louVxsQwDAYDVR0TBAUwAwEB/zAN
-RbEz/1ZeELz9KMwMpooVPIaYAWgqe+UNWuzHt0+jrH30NBcBv407G8eR/FWOU/cx
+BgkqhkiG9w0BAQUFAAOCAgEANxijK3JQNZnrDYv7E5Ny17EtxV6ADggs8BIFLHrp
-y/YMk3nXsAARoOcsFN1YSS1dNL1osezfsRStET8/bOEqpWD0yvt8wRWwh1hOCPVD
+tRISYw8HpFIrIF/MDbHgYGp/xkefGKGEHeS7rUPYwdAbKM0sfoxKXm5e8GGe9L5K
-OpWTZx7+dZcK1Zh64Rm5mPYzbhWYxGGqNuZGFCuR9yI2bHHsFI69LryUKNf1cJ/N
+pdG+ig1Ptm+Pae2Rcdj9RHKGmpAiKIF8a15l/Yj3jDVk06kx+lnT5fOePGhZBeuj
-dfvHt4GDxfF5ie4PWNgTp52wuI3YxNpsHgz9SmSEey6uVlA13vTO1QFX8Ymbyn6K
+duBZ2vP39rFfcBtTvFmoQRwfoa46fZEoWoXb3YwzBqIhBg9m80R+E79/HsRPwA4L
-FRhr2LHY4iBdY+Gw47WnAqdo7uXpyM3wT6jI4gn7oENvCSUyM/JMSQqE1Etw0LBr
+pOvcFTr28jNp1OadgZ92sY9EYabes23amebz/P6IOjutqssIdrPSKqM9aphlGLXE
-NIlC/RxN5wjcDvVCL/uS3PL6IW7R0wxrCQwBU3f5wMOnDM/R4EWJdS96zyb7Xnh3
+7YDxS9nSfX165Aa8NIWO95ivdbZplisnQ3rQM4pIdk7Z8FPhHftMdhekDREMxYKX
-PQGoj6/vllymI7tuwRhEuvFknRRihu3vilHgtGczVXTG73nFJftLzvN/OhqSSQG/
+KXepi5tLyVnhETj+ifYBwqxZ024rlnpnHUWgjxRz5atKTAsbAgcxHOYTKMZoRAod
-3c2JDX+vAy5jwPT/M3nPkrs68M4P77da1/BDZ0/KgJb/JzYZyNpq1nhWo3nMn+Sx
+BK7lvjZ7+C/cqUc2c9FSG/HxkrfMpJHJlzMsanTBJ1+MeUybeBtp5E7gdNALbfh/
-jq7y+h6ry8Omnlw7a/7CnNgvkLfP/uTfllL4erETFntHNh6LqCvpPNOqrvAP5keB
+BJ4eWw7X7q2oKape+7+OMX7aKAIysM7d2iVRuBofLBxOqzY6mzP8+Ro8zIgwFUeh
-EB8yoJraypfuiNELOw1zSRksMxe2ac4b/dhDNStBTPC0egfRSm3FA0XoOQ==
+r6pbEa8P2DXnuZ+PtcMiClYKuSLlf6xRRDMnHCxvsu1zA/Ga3vZ6g0bd487DIsGP
+tXHCYXttMGNxZDNVKS6rkrY2sT5xnJwvHwWmiooUZmSUFUdpqsvV5r9v89NMQ87L
+gNA=
 -----END CERTIFICATE-----
 '''
 

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (1, 27, 0)
+numeric_version = (1, 29, 0)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/db/backend.py

@@ -233,7 +233,7 @@ def AumSortedConcatenate():
 
 class Connection(apsw.Connection):  # {{{
 
-    BUSY_TIMEOUT = 2000  # milliseconds
+    BUSY_TIMEOUT = 10000  # milliseconds
 
     def __init__(self, path):
         apsw.Connection.__init__(self, path)

src/calibre/db/backup.py

@@ -93,6 +93,7 @@ class MetadataBackup(Thread):
         except:
             prints('Failed to convert to opf for id:', book_id)
             traceback.print_exc()
+            self.db.clear_dirtied(book_id, sequence)
             return
 
         self.wait(self.scheduling_interval)

src/calibre/db/cache.py

@@ -196,6 +196,7 @@ class Cache(object):
     def reload_from_db(self, clear_caches=True):
         if clear_caches:
             self._clear_caches()
+        with self.backend.conn:  # Prevent other processes, such as calibredb from interrupting the reload by locking the db
             self.backend.prefs.load_from_db()
             self._search_api.saved_searches.load_from_db()
             for field in self.fields.itervalues():

src/calibre/db/restore.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, os, traceback, shutil
+import re, os, traceback, shutil, time
 from threading import Thread
 from operator import itemgetter
 
@@ -269,7 +269,14 @@ class Restore(Thread):
         save_path = self.olddb = os.path.splitext(dbpath)[0]+'_pre_restore.db'
         if os.path.exists(save_path):
             os.remove(save_path)
+        try:
             os.rename(dbpath, save_path)
+        except OSError as err:
+            if getattr(err, 'winerror', None) == 32:  # ERROR_SHARING_VIOLATION
+                time.sleep(4)  # Wait a little for dropbox or the antivirus or whatever to release the file
+                os.rename(dbpath, save_path)
+            else:
+                raise
         shutil.copyfile(ndbpath, dbpath)
 
 

src/calibre/db/search.py

@@ -16,7 +16,7 @@ from calibre.constants import preferred_encoding
 from calibre.db.utils import force_to_bool
 from calibre.utils.config_base import prefs
 from calibre.utils.date import parse_date, UNDEFINED_DATE, now, dt_as_local
-from calibre.utils.icu import primary_find, sort_key
+from calibre.utils.icu import primary_contains, sort_key
 from calibre.utils.localization import lang_map, canonicalize_lang
 from calibre.utils.search_query_parser import SearchQueryParser, ParseException
 
@@ -73,7 +73,7 @@ def _match(query, value, matchkind, use_primary_find_in_search=True):
                     return True
             elif matchkind == CONTAINS_MATCH:
                 if use_primary_find_in_search:
-                    if primary_find(query, t)[0] != -1:
+                    if primary_contains(query, t):
                         return True
                 elif query in t:
                         return True

src/calibre/devices/idevice/libimobiledevice.py

@@ -184,7 +184,9 @@ class libiMobileDevice():
 
     def __init__(self, **kwargs):
         self.verbose = kwargs.get('verbose', False)
-
+        if not self.verbose:
+            self._log = self.__null
+            self._log_location = self.__null
         self._log_location()
         self.afc = None
         self.app_version = 0
@@ -230,7 +232,7 @@ class libiMobileDevice():
         src: file on local filesystem
         dst: file to be created on iOS filesystem
         '''
-        self._log_location("src=%s, dst=%s" % (repr(src), repr(dst)))
+        self._log_location("src:{0} dst:{1}".format(repr(src), repr(dst)))
         mode = 'rb'
         with open(src, mode) as f:
             content = bytearray(f.read())
@@ -239,7 +241,7 @@ class libiMobileDevice():
         handle = self._afc_file_open(str(dst), mode=mode)
         if handle is not None:
             success = self._afc_file_write(handle, content, mode=mode)
-            self._log(" success: %s" % success)
+            self._log(" success: {0}".format(success))
             self._afc_file_close(handle)
         else:
             self._log(" could not create copy")
@@ -251,7 +253,10 @@ class libiMobileDevice():
         src: path to file on iDevice
         dst: file object on local filesystem
         '''
-        self._log_location("src='%s', dst='%s'" % (src, dst.name))
+        self._log_location()
+        self._log("src: {0}".format(repr(src)))
+        self._log("dst: {0}".format(dst.name))
+
         BUFFER_SIZE = 10 * 1024 * 1024
         data = None
         mode = 'rb'
@@ -287,7 +292,7 @@ class libiMobileDevice():
 
         else:
             self._log(" could not open file")
-            raise libiMobileDeviceIOException("could not open file %s for reading" % repr(src))
+            raise libiMobileDeviceIOException("could not open file {0} for reading".format(repr(src)))
 
     def disconnect_idevice(self):
         '''
@@ -310,14 +315,14 @@ class libiMobileDevice():
             self._idevice_free()
             self.device_mounted = False
 
-    def exists(self, path):
+    def exists(self, path, silent=False):
         '''
         Determine if path exists
 
         Returns file_info or {}
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
-        return self._afc_get_file_info(path)
+        return self._afc_get_file_info(path, silent=silent)
 
     def get_device_info(self):
         '''
@@ -403,13 +408,13 @@ class libiMobileDevice():
         self._log_location()
         return self._lockdown_get_value(requested_items)
 
-    def listdir(self, path):
+    def listdir(self, path, get_stats=True):
         '''
         Return a list containing the names of the entries in the iOS directory
         given by path.
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
-        return self._afc_read_directory(path)
+        return self._afc_read_directory(path, get_stats=get_stats)
 
     def load_library(self):
         if islinux:
@@ -438,8 +443,8 @@ class libiMobileDevice():
             self.plist_lib = cdll.LoadLibrary('libplist.dll')
 
         self._log_location(env)
-        self._log(" libimobiledevice loaded from '%s'" % self.lib._name)
+        self._log(" libimobiledevice loaded from '{0}'".format(self.lib._name))
-        self._log(" libplist loaded from '%s'" % self.plist_lib._name)
+        self._log(" libplist loaded from '{0}'".format(self.plist_lib._name))
 
         if False:
             self._idevice_set_debug_level(DEBUG)
@@ -449,7 +454,7 @@ class libiMobileDevice():
         Mimic mkdir(), creating a directory at path. Does not create
         intermediate folders
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
         return self._afc_make_directory(path)
 
     def mount_ios_app(self, app_name=None, app_id=None):
@@ -481,7 +486,7 @@ class libiMobileDevice():
                 self._instproxy_client_free()
 
                 if not app_name in self.installed_apps:
-                    self._log(" '%s' not installed on this iDevice" % app_name)
+                    self._log(" {0} not installed on this iDevice".format(repr(app_name)))
                     self.disconnect_idevice()
                 else:
                     # Mount the app's Container
@@ -517,9 +522,9 @@ class libiMobileDevice():
                 self.disconnect_idevice()
 
         if self.device_mounted:
-            self._log_location("'%s' mounted" % (app_name if app_name else app_id))
+            self._log_location("'{0}' mounted".format(app_name if app_name else app_id))
         else:
-            self._log_location("unable to mount '%s'" % (app_name if app_name else app_id))
+            self._log_location("unable to mount '{0}'".format(app_name if app_name else app_id))
         return self.device_mounted
 
     def mount_ios_media_folder(self):
@@ -559,7 +564,7 @@ class libiMobileDevice():
         Use for small files.
         For larger files copied to local file, use copy_from_idevice()
         '''
-        self._log_location("'%s', mode='%s'" % (path, mode))
+        self._log_location("{0} mode='{1}'".format(repr(path), mode))
 
         data = None
         handle = self._afc_file_open(path, mode)
@@ -569,7 +574,7 @@ class libiMobileDevice():
             self._afc_file_close(handle)
         else:
             self._log(" could not open file")
-            raise libiMobileDeviceIOException("could not open file %s for reading" % repr(path))
+            raise libiMobileDeviceIOException("could not open file {0} for reading".format(repr(path)))
 
         return data
 
@@ -581,13 +586,13 @@ class libiMobileDevice():
         from_name:   (const char *) The fully-qualified path to rename from
         to_name:     (const char *) The fully-qualified path to rename to
         '''
-        self._log_location("from: '%s' to: '%s'" % (from_name, to_name))
+        self._log_location("from: {0} to: {1}".format(repr(from_name), repr(to_name)))
 
         error = self.lib.afc_rename_path(byref(self.afc),
                                          str(from_name),
                                          str(to_name))
         if error:
-            self._log(" ERROR: %s" % self._afc_error(error))
+            self._log(" ERROR: {0}".format(self._afc_error(error)))
 
     def remove(self, path):
         '''
@@ -596,12 +601,12 @@ class libiMobileDevice():
         client  (afc_client_t) The client to use
         path    (const char *) The fully-qualified path to delete
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
 
         error = self.lib.afc_remove_path(byref(self.afc), str(path))
 
         if error:
-            self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path)))
+            self._log_error(" ERROR: {0} path:{1}".format(self._afc_error(error), repr(path)))
 
     def stat(self, path):
         '''
@@ -615,19 +620,19 @@ class libiMobileDevice():
            'st_birthtime': xxx.yyy}
 
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
         return self._afc_get_file_info(path)
 
     def write(self, content, destination, mode='w'):
         '''
         Convenience method to write to path on iDevice
         '''
-        self._log_location(destination)
+        self._log_location("{0}".format(repr(destination)))
 
         handle = self._afc_file_open(destination, mode=mode)
         if handle is not None:
             success = self._afc_file_write(handle, content, mode=mode)
-            self._log(" success: %s" % success)
+            self._log(" success: {0}".format(success))
             self._afc_file_close(handle)
         else:
             self._log(" could not open file for writing")
@@ -650,7 +655,7 @@ class libiMobileDevice():
 
         error = self.lib.afc_client_free(byref(self.afc)) & 0xFFFF
         if error:
-            self._log_error(" ERROR: %s" % self._afc_error(error))
+            self._log_error(" ERROR: {0}".format(self._afc_error(error)))
 
     def _afc_client_new(self):
         '''
@@ -805,12 +810,12 @@ class libiMobileDevice():
          File closed
 
         '''
-        self._log_location(handle.value)
+        self._log_location("handle:{0}".format(handle.value))
 
         error = self.lib.afc_file_close(byref(self.afc),
                                         handle) & 0xFFFF
         if error:
-            self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle))
+            self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle))
 
     def _afc_file_open(self, filename, mode='r'):
         '''
@@ -834,7 +839,7 @@ class libiMobileDevice():
          error:      (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value
 
         '''
-        self._log_location("%s, mode='%s'" % (repr(filename), mode))
+        self._log_location("{0} mode='{1}'".format(repr(filename), mode))
 
         handle = c_ulonglong(0)
 
@@ -850,7 +855,7 @@ class libiMobileDevice():
                                            byref(handle)) & 0xFFFF
 
         if error:
-            self._log_error(" ERROR: %s filename:%s" % (self._afc_error(error), repr(filename)))
+            self._log_error(" ERROR: {0} filename:{1}".format(self._afc_error(error), repr(filename)))
             return None
         else:
             return handle
@@ -874,7 +879,7 @@ class libiMobileDevice():
          error       (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value
 
         '''
-        self._log_location("%s, size=%d, mode='%s'" % (handle.value, size, mode))
+        self._log_location("handle:{0} size:{1:,} mode='{2}'".format(handle.value, size, mode))
 
         bytes_read = c_uint(0)
 
@@ -887,13 +892,13 @@ class libiMobileDevice():
                                            size,
                                            byref(bytes_read)) & 0xFFFF
             if error:
-                self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle))
+                self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle))
             return data
         else:
             data = create_string_buffer(size)
             error = self.lib.afc_file_read(byref(self.afc), handle, byref(data), size, byref(bytes_read))
             if error:
-                self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle))
+                self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle))
             return data.value
 
     def _afc_file_write(self, handle, content, mode='w'):
@@ -915,7 +920,7 @@ class libiMobileDevice():
          error:         (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value
 
         '''
-        self._log_location("handle=%d, mode='%s'" % (handle.value, mode))
+        self._log_location("handle:{0} mode='{1}'".format(handle.value, mode))
 
         bytes_written = c_uint(0)
 
@@ -933,7 +938,7 @@ class libiMobileDevice():
                                         len(content),
                                         byref(bytes_written)) & 0xFFFF
         if error:
-            self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle))
+            self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle))
             return False
         return True
 
@@ -976,12 +981,12 @@ class libiMobileDevice():
                     for key in device_info.keys():
                         self._log("{0:>16}: {1}".format(key, device_info[key]))
             else:
-                self._log(" ERROR: %s" % self._afc_error(error))
+                self._log(" ERROR: {0}".format(self._afc_error(error)))
         else:
             self._log(" ERROR: AFC not initialized, can't get device info")
         return device_info
 
-    def _afc_get_file_info(self, path):
+    def _afc_get_file_info(self, path, silent=False):
         '''
         Gets information about a specific file
 
@@ -1003,7 +1008,7 @@ class libiMobileDevice():
            'st_birthtime': xxx.yyy}
 
         '''
-        self._log_location("'%s'" % path)
+        self._log_location("{0}".format(repr(path)))
 
         infolist_p = c_char * 1024
         infolist = POINTER(POINTER(infolist_p))()
@@ -1012,7 +1017,8 @@ class libiMobileDevice():
                                            byref(infolist)) & 0xFFFF
         file_stats = {}
         if error:
-            self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path)))
+            if not silent or self.verbose:
+                self._log_error(" ERROR: {0} path:{1}".format(self._afc_error(error), repr(path)))
         else:
             num_items = 0
             item_list = []
@@ -1023,14 +1029,14 @@ class libiMobileDevice():
                 if item_list[i].contents.value in ['st_mtime', 'st_birthtime']:
                     integer = item_list[i+1].contents.value[:10]
                     decimal = item_list[i+1].contents.value[10:]
-                    value = float("%s.%s" % (integer, decimal))
+                    value = float("{0}.{1}".format(integer, decimal))
                 else:
                     value = item_list[i+1].contents.value
                 file_stats[item_list[i].contents.value] = value
 
             if False and self.verbose:
                 for key in file_stats.keys():
-                    self._log(" %s: %s" % (key, file_stats[key]))
+                    self._log(" {0}: {1}".format(key, file_stats[key]))
         return file_stats
 
     def _afc_make_directory(self, path):
@@ -1044,16 +1050,16 @@ class libiMobileDevice():
         Result:
          error:  AFC_E_SUCCESS on success or an AFC_E_* error value
         '''
-        self._log_location("%s" % repr(path))
+        self._log_location("{0}".format(repr(path)))
 
         error = self.lib.afc_make_directory(byref(self.afc),
                                             str(path)) & 0xFFFF
         if error:
-            self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path)))
+            self._log_error(" ERROR: {0} path: {1}".format(self._afc_error(error), repr(path)))
 
         return error
 
-    def _afc_read_directory(self, directory=''):
+    def _afc_read_directory(self, directory='', get_stats=True):
         '''
         Gets a directory listing of the directory requested
 
@@ -1062,14 +1068,15 @@ class libiMobileDevice():
          dir:       (const char *) The directory to list (a fully-qualified path)
          list:      (char ***) A char list of files in that directory, terminated by
                      an empty string. NULL if there was an error.
-
+         get_stats: If True, return full file stats for each file in dir (slower)
+                    If False, return filename only (faster)
         Result:
          error: AFC_E_SUCCESS on success or an AFC_E_* error value
          file_stats:
             {'<path_basename>': {<file_stats>} ...}
 
         '''
-        self._log_location("'%s'" % directory)
+        self._log_location("{0}".format(repr(directory)))
 
         file_stats = {}
         dirs_p = c_char_p
@@ -1078,7 +1085,7 @@ class libiMobileDevice():
                                             str(directory),
                                             byref(dirs)) & 0xFFFF
         if error:
-            self._log_error(" ERROR: %s directory:%s" % (self._afc_error(error), repr(directory)))
+            self._log_error(" ERROR: {0} directory: {1}".format(self._afc_error(error), repr(directory)))
         else:
             num_dirs = 0
             dir_list = []
@@ -1094,7 +1101,10 @@ class libiMobileDevice():
                     path = '/' + this_item
                 else:
                     path = '/'.join([directory, this_item])
+                if get_stats:
                     file_stats[os.path.basename(path)] = self._afc_get_file_info(path)
+                else:
+                    file_stats[os.path.basename(path)] = {}
             self.current_dir = directory
         return file_stats
 
@@ -1126,7 +1136,7 @@ class libiMobileDevice():
         error = self.lib.house_arrest_client_free(byref(self.house_arrest)) & 0xFFFF
         if error:
             error = error - 0x10000
-            self._log_error(" ERROR: %s" % self._house_arrest_error(error))
+            self._log_error(" ERROR: {0}".format(self._house_arrest_error(error)))
 
     def _house_arrest_client_new(self):
         '''
@@ -1218,9 +1228,9 @@ class libiMobileDevice():
 
         # To determine success, we need to inspect the returned plist
         if 'Status' in result:
-            self._log("          STATUS: %s" % result['Status'])
+            self._log("          STATUS: {0}".format(result['Status']))
         elif 'Error' in result:
-            self._log("           ERROR: %s" % result['Error'])
+            self._log("           ERROR: {0}".format(result['Error']))
             raise libiMobileDeviceException(result['Error'])
 
     def _house_arrest_send_command(self, command=None, appid=None):
@@ -1244,12 +1254,12 @@ class libiMobileDevice():
                   to call house_arrest_get_result().
 
         '''
-        self._log_location("command='%s' appid='%s'" % (command, appid))
+        self._log_location("command={0} appid={1}".format(repr(command), repr(appid)))
 
         commands = ['VendContainer', 'VendDocuments']
 
         if command not in commands:
-            self._log(" ERROR: available commands: %s" % ', '.join(commands))
+            self._log(" ERROR: available commands: {0}".format(', '.join(commands)))
             return
 
         _command = create_string_buffer(command)
@@ -1302,7 +1312,7 @@ class libiMobileDevice():
 
         if error:
             error = error - 0x10000
-            self._log_error(" ERROR: %s" % self._idevice_error(error))
+            self._log_error(" ERROR: {0}".format(self._idevice_error(error)))
 
     def _idevice_get_device_list(self):
         '''
@@ -1326,7 +1336,7 @@ class libiMobileDevice():
                 self._log(" no connected devices")
             else:
                 device_list = None
-                self._log_error(" ERROR: %s" % self._idevice_error(error))
+                self._log_error(" ERROR: {0}".format(self._idevice_error(error)))
         else:
             index = 0
             while devices[index]:
@@ -1334,7 +1344,7 @@ class libiMobileDevice():
                 if devices[index].contents.value not in device_list:
                     device_list.append(devices[index].contents.value)
                 index += 1
-            self._log(" %s" % repr(device_list))
+            self._log(" {0}".format(repr(device_list)))
         #self.lib.idevice_device_list_free()
         return device_list
 
@@ -1368,8 +1378,8 @@ class libiMobileDevice():
             if idevice_t.contents.conn_type == 1:
                 self._log("       conn_type: CONNECTION_USBMUXD")
             else:
-                self._log("       conn_type: Unknown (%d)" % idevice_t.contents.conn_type)
+                self._log("       conn_type: Unknown ({0})".format(idevice_t.contents.conn_type))
-            self._log("            udid: %s" % idevice_t.contents.udid)
+            self._log("            udid: {0}".format(idevice_t.contents.udid))
             return idevice_t.contents
 
     def _idevice_set_debug_level(self, debug):
@@ -1406,7 +1416,7 @@ class libiMobileDevice():
         else:
             # Get the number of apps
             #app_count = self.lib.plist_array_get_size(apps)
-            #self._log("       app_count: %d" % app_count)
+            #self._log("       app_count: {0}".format(app_count))
 
             # Convert the app plist to xml
             xml = POINTER(c_void_p)()
@@ -1424,7 +1434,7 @@ class libiMobileDevice():
                 else:
                     self._log(" unable to find app name in bundle:")
                     for key in sorted(app.keys()):
-                        self._log("  %s   %s" % (repr(key), repr(app[key])))
+                        self._log("  {0}   {1}".format(repr(key), repr(app[key])))
                     continue
 
                 if not applist:
@@ -1483,7 +1493,7 @@ class libiMobileDevice():
         '''
         Specify the type of apps we want to browse
         '''
-        self._log_location("'%s', '%s'" % (app_type, domain))
+        self._log_location("{0}, {1}".format(repr(app_type), repr(domain)))
 
         self.lib.instproxy_client_options_add(self.client_options,
                                               app_type, domain, None)
@@ -1575,11 +1585,11 @@ class libiMobileDevice():
         self._log_location()
 
         lockdownd_client_t = POINTER(LOCKDOWND_CLIENT_T)()
-        SERVICE_NAME = create_string_buffer('calibre')
+        #SERVICE_NAME = create_string_buffer('calibre')
+        SERVICE_NAME = c_void_p()
         error = self.lib.lockdownd_client_new_with_handshake(byref(self.device),
                                                              byref(lockdownd_client_t),
                                                              SERVICE_NAME) & 0xFFFF
-
         if error:
             error = error - 0x10000
             error_description = self.LIB_ERROR_TEMPLATE.format(
@@ -1649,8 +1659,7 @@ class libiMobileDevice():
         '''
         self._log_location()
 
-        device_name_b = c_char * 32
+        device_name_p = c_char_p()
-        device_name_p = POINTER(device_name_b)()
         device_name = None
         error = self.lib.lockdownd_get_device_name(byref(self.control),
                                                    byref(device_name_p)) & 0xFFFF
@@ -1662,8 +1671,8 @@ class libiMobileDevice():
                 desc=self._lockdown_error(error))
             raise libiMobileDeviceException(error_description)
         else:
-            device_name = device_name_p.contents.value
+            device_name = device_name_p.value
-            self._log("     device_name: %s" % device_name)
+            self._log("     device_name: {0}".format(device_name))
         return device_name
 
     def _lockdown_get_value(self, requested_items=[]):
@@ -1811,7 +1820,7 @@ class libiMobileDevice():
         if self.control:
             error = self.lib.lockdownd_goodbye(byref(self.control)) & 0xFFFF
             error = error - 0x10000
-            self._log(" ERROR: %s" % self.error_lockdown(error))
+            self._log(" ERROR: {0}".format(self.error_lockdown(error)))
         else:
             self._log(" connection already closed")
 
@@ -1851,11 +1860,8 @@ class libiMobileDevice():
         '''
         Print msg to console
         '''
-        if not self.verbose:
-            return
-
         if msg:
-            debug_print(" %s" % msg)
+            debug_print(" {0}".format(msg))
         else:
             debug_print()
 
@@ -1876,9 +1882,6 @@ class libiMobileDevice():
     def _log_location(self, *args):
         '''
         '''
-        if not self.verbose:
-            return
-
         arg1 = arg2 = ''
 
         if len(args) > 0:
@@ -1888,3 +1891,6 @@ class libiMobileDevice():
 
         debug_print(self.LOCATION_TEMPLATE.format(cls=self.__class__.__name__,
             func=sys._getframe(1).f_code.co_name, arg1=arg1, arg2=arg2))
+
+    def __null(self, *args, **kwargs):
+        pass

src/calibre/devices/kobo/driver.py

@@ -68,7 +68,7 @@ class KOBO(USBMS):
 
     dbversion = 0
     fwversion = 0
-    supported_dbversion = 95
+    supported_dbversion = 98
     has_kepubs = False
 
     supported_platforms = ['windows', 'osx', 'linux']

src/calibre/devices/nokia/driver.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 __license__   = 'GPL v3'
-__copyright__ = '2009, John Schember <john at nachtimwald.com>'
+__copyright__ = '2009-2014, John Schember <john at nachtimwald.com> and Andres Gomez <agomez at igalia.com>'
 __docformat__ = 'restructuredtext en'
 
 '''
@@ -14,13 +14,13 @@ class N770(USBMS):
 
     name           = 'Nokia 770 Device Interface'
     gui_name       = 'Nokia 770'
-    description    = _('Communicate with the Nokia 770 internet tablet.')
+    description    = _('Communicate with the Nokia 770 Internet Tablet.')
-    author         = 'John Schember'
+    author         = 'John Schember and Andres Gomez'
     supported_platforms = ['windows', 'linux', 'osx']
 
     # Ordered list of supported formats
-    FORMATS     = ['mobi', 'prc', 'epub', 'html', 'zip', 'fb2', 'chm', 'pdb',
+    FORMATS     = ['mobi', 'prc', 'epub', 'pdf', 'html', 'zip', 'fb2', 'chm',
-        'tcr', 'txt', 'rtf']
+        'pdb', 'tcr', 'txt', 'rtf']
 
     VENDOR_ID   = [0x421]
     PRODUCT_ID  = [0x431]
@@ -29,22 +29,22 @@ class N770(USBMS):
     VENDOR_NAME      = 'NOKIA'
     WINDOWS_MAIN_MEM = '770'
 
-    MAIN_MEMORY_VOLUME_LABEL  = 'N770 Main Memory'
+    MAIN_MEMORY_VOLUME_LABEL  = 'Nokia 770 Main Memory'
 
     EBOOK_DIR_MAIN = 'My Ebooks'
     SUPPORTS_SUB_DIRS = True
 
 class N810(N770):
-    name           = 'Nokia 810 Device Interface'
+    name           = 'Nokia N800/N810/N900/N950/N9 Device Interface'
-    gui_name       = 'Nokia 810/900/9'
+    gui_name       = 'Nokia N800/N810/N900/N950/N9'
-    description    = _('Communicate with the Nokia 810/900 internet tablet.')
+    description    = _('Communicate with the Nokia N800/N810/N900/N950/N9 Maemo/MeeGo devices.')
 
-    PRODUCT_ID = [0x96, 0x1c7, 0x0518]
+    PRODUCT_ID = [0x4c3, 0x96, 0x1c7, 0x3d1, 0x518]
     BCD        = [0x316]
 
-    WINDOWS_MAIN_MEM = ['N810', 'N900', 'NOKIA_N9']
+    WINDOWS_MAIN_MEM = ['N800', 'N810', 'N900', 'NOKIA_N950', 'NOKIA_N9']
 
-    MAIN_MEMORY_VOLUME_LABEL = 'Nokia Tablet Main Memory'
+    MAIN_MEMORY_VOLUME_LABEL = 'Nokia Maemo/MeeGo device Main Memory'
 
 class E71X(USBMS):
 

src/calibre/devices/smart_device_app/driver.py

@@ -226,7 +226,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     PURGE_CACHE_ENTRIES_DAYS    = 30
 
-    CURRENT_CC_VERSION          = 64
+    CURRENT_CC_VERSION          = 73
 
     ZEROCONF_CLIENT_STRING      = b'calibre wireless device client'
 
@@ -1223,6 +1223,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     books_on_device.append(result)
 
                 books_to_send = []
+                lpaths_on_device = set()
                 for r in books_on_device:
                     if r.get('lpath', None):
                         book = self._metadata_in_cache(r['uuid'], r['lpath'],
@@ -1231,13 +1232,31 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                         book = self._metadata_in_cache(r['uuid'], r['extension'],
                                                        r['last_modified'])
                     if book:
+                        if self.client_cache_uses_lpaths:
+                            lpaths_on_device.add(r.get('lpath'))
                         bl.add_book(book, replace_metadata=True)
                         book.set('_is_read_', r.get('_is_read_', None))
-                        book.set('_is_read_changed_', r.get('_is_read_changed_', None))
+                        book.set('_sync_type_', r.get('_sync_type_', None))
                         book.set('_last_read_date_', r.get('_last_read_date_', None))
                     else:
                         books_to_send.append(r['priKey'])
 
+                count_of_cache_items_deleted = 0
+                if self.client_cache_uses_lpaths:
+                    for lpath in self.known_metadata.keys():
+                        if lpath not in lpaths_on_device:
+                            try:
+                                uuid = self.known_metadata[lpath].get('uuid', None)
+                                if uuid is not None:
+                                    key = self._make_metadata_cache_key(uuid, lpath)
+                                    self.device_book_cache.pop(key, None)
+                                    self.known_metadata.pop(lpath, None)
+                                    count_of_cache_items_deleted += 1
+                            except:
+                                self._debug('Exception while deleting book from caches', lpath)
+                                traceback.print_exc()
+                    self._debug('removed', count_of_cache_items_deleted, 'books from caches')
+
                 count = len(books_to_send)
                 self._debug('caching. Need count from device', count)
 
@@ -1256,7 +1275,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                         del result['_series_sort_']
                     book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX)
                     book.set('_is_read_', result.get('_is_read_', None))
-                    book.set('_is_read_changed_', result.get('_is_read_changed_', None))
+                    book.set('_sync_type_', result.get('_sync_type_', None))
                     book.set('_last_read_date_', result.get('_last_read_date_', None))
                     bl.add_book(book, replace_metadata=True)
                     if '_new_book_' in result:
@@ -1512,93 +1531,137 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if self.have_bad_sync_columns:
                 return None
 
-        is_changed = book.get('_is_read_changed_', None);
+        sync_type = book.get('_sync_type_', None);
+        # We need to check if our attributes are in the book. If they are not
+        # then this is metadata coming from calibre to the device for the first
+        # time, in which case we must not sync it.
+        if hasattr(book, '_is_read_'):
             is_read = book.get('_is_read_', None)
+            has_is_read = True
+        else:
+            has_is_read = False
 
-        # This returns UNDEFINED_DATE if the value is None
+        if hasattr(book, '_last_read_date_'):
+            # parse_date returns UNDEFINED_DATE if the value is None
             is_read_date = parse_date(book.get('_last_read_date_', None));
             if is_date_undefined(is_read_date):
                 is_read_date = None
+            has_is_read_date = True
+        else:
+            has_is_read_date = False
 
-        value_to_return = None
+        force_return_changed_books = False
+        changed_books = set()
 
-        if is_changed == 2:
+        if sync_type == 3:
-            # This is a special case where the user just set the sync column. In
+            # The book metadata was built by the device from metadata in the
-            # this case the device value wins if it is not None by falling
+            # book file itself. It must not be synced, because the metadata is
-            # through to the normal sync situation below, otherwise the calibre
+            # almost surely wrong. However, the fact that we got here means that
-            # value wins. The orig_* values are set to None to force the normal
+            # book matching has succeeded. Arrange that calibre's metadata is
-            # sync code to actually sync because the values are different
+            # sent back to the device. This isn't strictly necessary as sending
-            orig_is_read_date = None
+            # back the info will be arranged in other ways.
-            orig_is_read = None
+            self._debug('Book with device-generated metadata', book.get('title', 'huh?'))
+            book.set('_force_send_metadata_', True)
+            force_return_changed_books = True
+        elif sync_type == 2:
+            # This is a special case where the user just set a sync column. In
+            # this case the device value wins if it is not None, otherwise the
+            # calibre value wins.
 
-            if is_read is None:
+            # Check is_read
+            if has_is_read and self.is_read_sync_col:
+                try:
                     calibre_val = db.new_api.field_for(self.is_read_sync_col,
                                                        id_, default_value=None)
-                if calibre_val is not None:
+                    if is_read is not None:
-                    # This forces the metadata for the book to be sent to the
+                        # The CC value wins. Check if it is different from calibre's
-                    # device even if the mod dates haven't changed.
+                        # value to avoid updating the db to the same value
+                        if is_read != calibre_val:
+                            self._debug('special update calibre to is_read',
+                                    book.get('title', 'huh?'), 'to', is_read, calibre_val)
+                            changed_books = db.new_api.set_field(self.is_read_sync_col,
+                                                                 {id_: is_read})
+                    elif calibre_val is not None:
+                        # Calibre value wins. Force the metadata for the
+                        # book to be sent to the device even if the mod
+                        # dates haven't changed.
+                        self._debug('special update is_read to calibre value',
+                                    book.get('title', 'huh?'), 'to', calibre_val)
                         book.set('_force_send_metadata_', True)
-                    self._debug('special update is_read', book.get('title', 'huh?'),
+                        force_return_changed_books = True
-                                'to', calibre_val)
+                except:
-                    value_to_return = set()
+                    self._debug('exception special syncing is_read', self.is_read_sync_col)
+                    traceback.print_exc()
 
-            if is_read_date is None:
+            # Check is_read_date.
+            if has_is_read_date and self.is_read_date_sync_col:
+                try:
+                    # The db method returns None for undefined dates.
                     calibre_val = db.new_api.field_for(self.is_read_date_sync_col,
                                                            id_, default_value=None)
-                if not is_date_undefined(calibre_val):
+                    if is_read_date is not None:
+                        if is_read_date != calibre_val:
+                            self._debug('special update calibre to is_read_date',
+                                book.get('title', 'huh?'), 'to', is_read_date, calibre_val)
+                            changed_books |= db.new_api.set_field(self.is_read_date_sync_col,
+                                                                 {id_: is_read_date})
+                    elif calibre_val is not None:
+                        self._debug('special update is_read_date to calibre value',
+                                    book.get('title', 'huh?'), 'to', calibre_val)
                         book.set('_force_send_metadata_', True)
-                    self._debug('special update is_read_date', book.get('title', 'huh?'),
+                        force_return_changed_books = True
-                                'to', calibre_val)
+                except:
-                    value_to_return = set()
+                    self._debug('exception special syncing is_read_date',
-            # Fall through to the normal sync. At this point either the is_read*
+                                self.is_read_sync_col)
-            # values are different from the orig_is_read* which will cause a
+                    traceback.print_exc()
-            # sync below, or they are both None which will cause the code below
-            # to do nothing. If either of the calibre data fields were set, the
-            # method will return set(), which will force updated metadata to be
-            # given back to the device, effectively forcing the sync of the
-            # calibre values back to the device.
         else:
-            orig_is_read = book.get(self.is_read_sync_col, None)
+            # This is the standard sync case. If the CC value has changed, it
-            orig_is_read_date = book.get(self.is_read_date_sync_col, None)
+            # wins, otherwise the calibre value is synced to CC in the normal
-
+            # fashion (mod date)
-        changed_books = set()
+            if has_is_read and self.is_read_sync_col:
                 try:
+                    orig_is_read = book.get(self.is_read_sync_col, None)
                     if is_read != orig_is_read:
-                # The value in the device's is_read checkbox is not the same as the
+                        # The value in the device's is_read checkbox is not the
-                # last one that came to the device from calibre during the last
+                        # same as the last one that came to the device from
-                # connect, meaning that the user changed it. Write the one from the
+                        # calibre during the last connect, meaning that the user
-                # device to calibre's db.
+                        # changed it. Write the one from the device to calibre's
-                self._debug('standard update book is_read', book.get('title', 'huh?'),
+                        # db.
-                            'to', is_read)
+                        self._debug('standard update is_read', book.get('title', 'huh?'),
-                if self.is_read_sync_col:
+                                    'to', is_read, 'was', orig_is_read)
                         changed_books = db.new_api.set_field(self.is_read_sync_col,
                                                                  {id_: is_read})
                 except:
-            self._debug('exception syncing is_read col', self.is_read_sync_col)
+                    self._debug('exception standard syncing is_read', self.is_read_sync_col)
                     traceback.print_exc()
 
+            if has_is_read_date and self.is_read_date_sync_col:
                 try:
+                    orig_is_read_date = book.get(self.is_read_date_sync_col, None)
+                    if is_date_undefined(orig_is_read_date):
+                        orig_is_read_date = None
+
                     if is_read_date != orig_is_read_date:
-                self._debug('standard update book is_read_date', book.get('title', 'huh?'),
+                        self._debug('standard update is_read_date', book.get('title', 'huh?'),
                                     'to', is_read_date, 'was', orig_is_read_date)
-                if self.is_read_date_sync_col:
                         changed_books |= db.new_api.set_field(self.is_read_date_sync_col,
                                                           {id_: is_read_date})
                 except:
-            self._debug('Exception while syncing is_read_date', self.is_read_date_sync_col)
+                    self._debug('Exception standard syncing is_read_date',
+                                self.is_read_date_sync_col)
                     traceback.print_exc()
 
-        if changed_books:
+        if changed_books or force_return_changed_books:
-            # One of the two values was synced, giving a list of changed books.
+            # One of the two values was synced, giving a (perhaps empty) list of
-            # Return that.
+            # changed books. Return that.
             return changed_books
 
-        # The user might have changed the value in calibre. If so, that value
+        # Nothing was synced. The user might have changed the value in calibre.
-        # will be sent to the device in the normal way. Note that because any
+        # If so, that value will be sent to the device in the normal way. Note
-        # updated value has already been synced and so will also be sent, the
+        # that because any updated value has already been synced and so will
-        # device should put the calibre value into its checkbox (or whatever it
+        # also be sent, the device should put the calibre value into its
-        # uses)
+        # checkbox (or whatever it uses)
-        return value_to_return
+        return None
 
     @synchronous('sync_lock')
     def startup(self):
@@ -1735,6 +1798,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def shutdown(self):
+        self._close_device_socket()
         if getattr(self, 'listen_socket', None) is not None:
             self.connection_listener.stop()
             try:

src/calibre/ebooks/docx/cleanup.py

@@ -171,7 +171,7 @@ def cleanup_markup(log, root, styles, dest_dir, detect_cover):
             if prefix:
                 prefix += '; '
             p.set('style', prefix + 'page-break-after:always')
-            p.text = NBSP
+            p.text = NBSP if not p.text else p.text
 
     if detect_cover:
         # Check if the first image in the document is possibly a cover

src/calibre/ebooks/metadata/book/render.py

@@ -148,7 +148,7 @@ def mi_to_html(mi, field_list=None, default_author_link=None, use_roman_numbers=
                             default_author_link, vals, '', vals)
                 aut = p(aut)
                 if link:
-                    authors.append(u'<a calibre-data="authors" href="%s">%s</a>'%(a(link), aut))
+                    authors.append(u'<a calibre-data="authors" title="%s" href="%s">%s</a>'%(a(link), a(link), aut))
                 else:
                     authors.append(aut)
             ans.append((field, row % (name, u' & '.join(authors))))

src/calibre/ebooks/metadata/meta.py

@@ -100,27 +100,17 @@ def _get_metadata(stream, stream_type, use_libprs_metadata,
     if use_libprs_metadata and getattr(opf, 'application_id', None) is not None:
         return opf
 
-    mi = MetaInformation(None, None)
     name = os.path.basename(getattr(stream, 'name', ''))
-    base = metadata_from_filename(name, pat=pattern)
+    # The fallback pattern matches the default filename format produced by calibre
-    if force_read_metadata or prefs['read_file_metadata']:
+    base = metadata_from_filename(name, pat=pattern, fallback_pat=re.compile(
-        mi = get_file_type_metadata(stream, stream_type)
+            r'^(?P<title>.+) - (?P<author>[^-]+)$'))
-    if base.title == os.path.splitext(name)[0] and \
-            base.is_null('authors') and base.is_null('isbn'):
-        # Assume that there was no metadata in the file and the user set pattern
-        # to match meta info from the file name did not match.
-        # The regex is meant to match the standard format filenames are written
-        # in the library title - author.extension
-        base.smart_update(metadata_from_filename(name, re.compile(
-            r'^(?P<title>.+)[ _]-[ _](?P<author>[^-]+)$')))
-        if base.title:
-            base.title = base.title.replace('_', ' ')
-        if base.authors:
-            base.authors = [a.replace('_', ' ').strip() for a in base.authors]
     if not base.authors:
         base.authors = [_('Unknown')]
     if not base.title:
         base.title = _('Unknown')
+    mi = MetaInformation(None, None)
+    if force_read_metadata or prefs['read_file_metadata']:
+        mi = get_file_type_metadata(stream, stream_type)
     base.smart_update(mi)
     if opf is not None:
         base.smart_update(opf)
@@ -133,7 +123,7 @@ def set_metadata(stream, mi, stream_type='lrf'):
     set_file_type_metadata(stream, mi, stream_type)
 
 
-def metadata_from_filename(name, pat=None):
+def metadata_from_filename(name, pat=None, fallback_pat=None):
     if isbytestring(name):
         name = name.decode(filesystem_encoding, 'replace')
     name = name.rpartition('.')[0]
@@ -142,6 +132,8 @@ def metadata_from_filename(name, pat=None):
         pat = re.compile(prefs.get('filename_pattern'))
     name = name.replace('_', ' ')
     match = pat.search(name)
+    if match is None and fallback_pat is not None:
+        match = fallback_pat.search(name)
     if match is not None:
         try:
             mi.title = match.group('title')

src/calibre/ebooks/metadata/toc.py

@@ -13,6 +13,7 @@ from lxml.builder import ElementMaker
 from calibre.constants import __appname__, __version__
 from calibre.ebooks.BeautifulSoup import BeautifulSoup
 from calibre.ebooks.chardet import xml_to_unicode
+from calibre.utils.cleantext import clean_xml_chars
 
 NCX_NS = "http://www.daisy.org/z3986/2005/ncx/"
 CALIBRE_NS = "http://calibre.kovidgoyal.net/2009/metadata"
@@ -258,6 +259,7 @@ class TOC(list):
                 text = ''
             c[1] += 1
             item_id = 'num_%d'%c[1]
+            text = clean_xml_chars(text)
             elem = E.navPoint(
                     E.navLabel(E.text(re.sub(r'\s+', ' ', text))),
                     E.content(src=unicode(np.href)+(('#' + unicode(np.fragment))

src/calibre/ebooks/oeb/display/full_screen.coffee

@@ -25,9 +25,10 @@ class FullScreen
         this.initial_left_margin = bs.marginLeft
         this.initial_right_margin = bs.marginRight
 
-    on: (max_text_width, in_paged_mode) ->
+    on: (max_text_width, max_text_height, in_paged_mode) ->
         if in_paged_mode
             window.paged_display.max_col_width = max_text_width
+            window.paged_display.max_col_height = max_text_height
         else
             s = document.body.style
             s.maxWidth = max_text_width + 'px'
@@ -39,6 +40,7 @@ class FullScreen
         window.removeEventListener('click', this.handle_click, false)
         if in_paged_mode
             window.paged_display.max_col_width = -1
+            window.paged_display.max_col_height = -1
         else
             s = document.body.style
             s.maxWidth = 'none'

src/calibre/ebooks/oeb/display/paged.coffee

@@ -8,6 +8,10 @@
 
 log = window.calibre_utils.log
 
+runscripts = (parent) ->
+    for script in parent.getElementsByTagName('script')
+        eval(script.text || script.textContent || script.innerHTML || '')
+
 class PagedDisplay
     # This class is a namespace to expose functions via the
     # window.paged_display object. The most important functions are:
@@ -22,10 +26,12 @@ class PagedDisplay
         this.set_geometry()
         this.page_width = 0
         this.screen_width = 0
+        this.side_margin = 0
         this.in_paged_mode = false
         this.current_margin_side = 0
         this.is_full_screen_layout = false
         this.max_col_width = -1
+        this.max_col_height = - 1
         this.current_page_height = null
         this.document_margins = null
         this.use_document_margins = false
@@ -71,10 +77,14 @@ class PagedDisplay
             this.margin_top = this.document_margins.top or margin_top
             this.margin_bottom = this.document_margins.bottom or margin_bottom
             this.margin_side = this.document_margins.left or this.document_margins.right or margin_side
+            this.effective_margin_top = this.margin_top
+            this.effective_margin_bottom = this.margin_bottom
         else
             this.margin_top = margin_top
             this.margin_side = margin_side
             this.margin_bottom = margin_bottom
+            this.effective_margin_top = this.margin_top
+            this.effective_margin_bottom = this.margin_bottom
 
     handle_rtl_body: (body_style) ->
         if body_style.direction == "rtl"
@@ -117,8 +127,8 @@ class PagedDisplay
             col_width = Math.max(100, ((ww - adjust)/n) - 2*sm)
         this.col_width = col_width
         this.page_width = col_width + 2*sm
+        this.side_margin = sm
         this.screen_width = this.page_width * this.cols_per_screen
-        this.current_page_height = window.innerHeight - this.margin_top - this.margin_bottom
 
         fgcolor = body_style.getPropertyValue('color')
 
@@ -142,12 +152,20 @@ class PagedDisplay
         if c?.nodeType == 1
             c.style.setProperty('-webkit-margin-before', '0')
 
+        this.effective_margin_top = this.margin_top
+        this.effective_margin_bottom = this.margin_bottom
+        this.current_page_height = window.innerHeight - this.margin_top - this.margin_bottom
+        if this.max_col_height > 0 and this.current_page_height > this.max_col_height
+            eh = Math.ceil((this.current_page_height - this.max_col_height) / 2)
+            this.effective_margin_top += eh
+            this.effective_margin_bottom += eh
+            this.current_page_height -= 2 * eh
 
         bs.setProperty('overflow', 'visible')
-        bs.setProperty('height', (window.innerHeight - this.margin_top - this.margin_bottom) + 'px')
+        bs.setProperty('height', this.current_page_height + 'px')
         bs.setProperty('width', (window.innerWidth - 2*sm)+'px')
-        bs.setProperty('margin-top', this.margin_top + 'px')
+        bs.setProperty('margin-top', this.effective_margin_top + 'px')
-        bs.setProperty('margin-bottom', this.margin_bottom+'px')
+        bs.setProperty('margin-bottom', this.effective_margin_bottom+'px')
         bs.setProperty('margin-left', sm+'px')
         bs.setProperty('margin-right', sm+'px')
         for edge in ['left', 'right', 'top', 'bottom']
@@ -193,12 +211,12 @@ class PagedDisplay
     create_header_footer: (uuid) ->
         if this.header_template != null
             this.header = document.createElement('div')
-            this.header.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: 0px; height: #{ this.margin_top }px; width: #{ this.col_width }px; margin: 0; padding: 0")
+            this.header.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: 0px; height: #{ this.effective_margin_top }px; width: #{ this.col_width }px; margin: 0; padding: 0")
             this.header.setAttribute('id', 'pdf_page_header_'+uuid)
             document.body.appendChild(this.header)
         if this.footer_template != null
             this.footer = document.createElement('div')
-            this.footer.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: #{ window.innerHeight - this.margin_bottom }px; height: #{ this.margin_bottom }px; width: #{ this.col_width }px; margin: 0; padding: 0")
+            this.footer.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: #{ window.innerHeight - this.effective_margin_bottom }px; height: #{ this.effective_margin_bottom }px; width: #{ this.col_width }px; margin: 0; padding: 0")
             this.footer.setAttribute('id', 'pdf_page_footer_'+uuid)
             document.body.appendChild(this.footer)
         if this.header != null or this.footer != null
@@ -224,8 +242,10 @@ class PagedDisplay
             section = py_bridge.section()
         if this.header != null
             this.header.innerHTML = this.header_template.replace(/_PAGENUM_/g, pagenum+"").replace(/_TITLE_/g, title+"").replace(/_AUTHOR_/g, author+"").replace(/_SECTION_/g, section+"")
+            runscripts(this.header)
         if this.footer != null
             this.footer.innerHTML = this.footer_template.replace(/_PAGENUM_/g, pagenum+"").replace(/_TITLE_/g, title+"").replace(/_AUTHOR_/g, author+"").replace(/_SECTION_/g, section+"")
+            runscripts(this.footer)
 
     fit_images: () ->
         # Ensure no images are wider than the available width in a column. Note
@@ -501,8 +521,8 @@ class PagedDisplay
                     continue
                 deltax = Math.floor(this.page_width/25)
                 deltay = Math.floor(window.innerHeight/25)
-                cury = this.margin_top
+                cury = this.effective_margin_top
-                until cury >= (window.innerHeight - this.margin_bottom)
+                until cury >= (window.innerHeight - this.effective_margin_bottom)
                     curx = left + this.current_margin_side
                     until curx >= (right - this.current_margin_side)
                         cfi = window.cfi.at_point(curx-window.pageXOffset, cury-window.pageYOffset)

src/calibre/ebooks/oeb/polish/check/main.py

@@ -50,7 +50,7 @@ def run_checks(container):
     for name, mt, raw in html_items:
         root = container.parsed(name)
         for style in root.xpath('//*[local-name()="style"]'):
-            if style.get('type', 'text/css') == 'text/css':
+            if style.get('type', 'text/css') == 'text/css' and style.text:
                 errors.extend(check_css_parsing(name, style.text, line_offset=style.sourceline - 1))
         for elem in root.xpath('//*[@style]'):
             raw = elem.get('style')

src/calibre/ebooks/oeb/polish/opf.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from lxml import etree
+
+from calibre.ebooks.oeb.polish.container import OPF_NAMESPACES
+from calibre.utils.localization import canonicalize_lang
+
+def get_book_language(container):
+    for lang in container.opf_xpath('//dc:language'):
+        raw = lang.text
+        if raw:
+            code = canonicalize_lang(raw.split(',')[0].strip())
+            if code:
+                return code
+
+def set_guide_item(container, item_type, title, name, frag=None):
+    ref_tag = '{%s}reference' % OPF_NAMESPACES['opf']
+    href = None
+    if name:
+        href = container.name_to_href(name, container.opf_name)
+        if frag:
+            href += '#' + frag
+
+    guides = container.opf_xpath('//opf:guide')
+    if not guides and href:
+        g = container.opf.makeelement('{%s}guide' % OPF_NAMESPACES['opf'], nsmap={'opf':OPF_NAMESPACES['opf']})
+        container.insert_into_xml(container.opf, g)
+        guides = [g]
+
+    for guide in guides:
+        matches = []
+        for child in guide.iterchildren(etree.Element):
+            if child.tag == ref_tag and child.get('type', '').lower() == item_type.lower():
+                matches.append(child)
+        if not matches and href:
+            r = guide.makeelement(ref_tag, type=item_type, nsmap={'opf':OPF_NAMESPACES['opf']})
+            container.insert_into_xml(guide, r)
+            matches.append(r)
+        for m in matches:
+            if href:
+                m.set('title', title), m.set('href', href), m.set('type', item_type)
+            else:
+                container.remove_from_xml(m)
+    container.dirty(container.opf_name)
+

src/calibre/ebooks/oeb/polish/pretty.py

@@ -93,7 +93,7 @@ BLOCK_TAGS = frozenset(map(XHTML, (
     'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form',
     'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li',
     'noscript', 'ol', 'output', 'p', 'pre', 'script', 'section', 'style', 'table', 'tbody', 'td',
-    'tfoot', 'thead', 'tr', 'ul', 'video'))) | {SVG_TAG}
+    'tfoot', 'thead', 'tr', 'ul', 'video', 'img'))) | {SVG_TAG}
 
 
 def isblock(x):

src/calibre/ebooks/oeb/polish/toc.py

@@ -20,7 +20,9 @@ from calibre import __version__
 from calibre.ebooks.oeb.base import XPath, uuid_id, xml2text, NCX, NCX_NS, XML, XHTML, XHTML_NS, serialize
 from calibre.ebooks.oeb.polish.errors import MalformedMarkup
 from calibre.ebooks.oeb.polish.utils import guess_type
+from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language
 from calibre.ebooks.oeb.polish.pretty import pretty_html_tree
+from calibre.translations.dynamic import translate
 from calibre.utils.localization import get_lang, canonicalize_lang, lang_as_iso639_1
 
 ns = etree.FunctionNamespace('calibre_xpath_extensions')
@@ -182,7 +184,7 @@ def find_existing_toc(container):
 
 def get_toc(container, verify_destinations=True):
     toc = find_existing_toc(container)
-    if toc is None:
+    if toc is None or not container.has_name(toc):
         ans = TOC()
         ans.lang = ans.uid = None
         return ans
@@ -481,7 +483,12 @@ def find_inline_toc(container):
             return name
 
 def create_inline_toc(container, title=None):
-    title = title or _('Table of Contents')
+    lang = get_book_language(container)
+    default_title = 'Table of Contents'
+    if lang:
+        lang = lang_as_iso639_1(lang) or lang
+        default_title = translate(lang, default_title)
+    title = title or default_title
     toc = get_toc(container)
     if len(toc) == 0:
         return None
@@ -529,6 +536,8 @@ def create_inline_toc(container, title=None):
     name = toc_name
     for child in toc:
         process_node(html[1][1], child)
+    if lang:
+        html.set('lang', lang)
     pretty_html_tree(container, html)
     raw = serialize(html, 'text/html')
     if name is None:
@@ -540,5 +549,6 @@ def create_inline_toc(container, title=None):
     else:
         with container.open(name, 'wb') as f:
             f.write(raw)
+    set_guide_item(container, 'toc', title, name, frag='calibre_generated_inline_toc')
     return name
 

src/calibre/ebooks/pdf/render/from_html.py

@@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en'
 import json, os
 from future_builtins import map
 from math import floor
+from collections import defaultdict
 
 from PyQt4.Qt import (QObject, QPainter, Qt, QSize, QString, QTimer,
                       pyqtProperty, QEventLoop, QPixmap, QRect, pyqtSlot)
@@ -310,7 +311,7 @@ class PDFWriter(QObject):
             evaljs('document.getElementById("MathJax_Message").style.display="none";')
 
     def get_sections(self, anchor_map):
-        sections = {}
+        sections = defaultdict(list)
         ci = os.path.abspath(os.path.normcase(self.current_item))
         if self.toc is not None:
             for toc in self.toc.flat():
@@ -323,8 +324,7 @@ class PDFWriter(QObject):
                     col = 0
                     if frag and frag in anchor_map:
                         col = anchor_map[frag]['column']
-                    if col not in sections:
+                    sections[col].append(toc.text or _('Untitled'))
-                        sections[col] = toc.text or _('Untitled')
 
         return sections
 
@@ -380,7 +380,11 @@ class PDFWriter(QObject):
         mf = self.view.page().mainFrame()
         while True:
             if col in sections:
-                self.current_section = sections[col]
+                self.current_section = sections[col][0]
+            elif col - 1 in sections:
+                # Ensure we are using the last section on the previous page as
+                # the section for this page, since this page has no sections
+                self.current_section = sections[col-1][-1]
             self.doc.init_page()
             if self.header or self.footer:
                 evaljs('paged_display.update_header_footer(%d)'%self.current_page_num)

src/calibre/gui2/__init__.py

@@ -123,6 +123,7 @@ defs['cover_grid_texture'] = None
 defs['show_vl_tabs'] = False
 defs['show_highlight_toggle_button'] = False
 defs['add_comments_to_email'] = False
+defs['cb_preserve_aspect_ratio'] = False
 del defs
 # }}}
 

src/calibre/gui2/add.py

@@ -424,7 +424,7 @@ class Adder(QObject):  # {{{
             return self.duplicates_processed()
         self.pd.hide()
         from calibre.gui2.dialogs.duplicates import DuplicatesQuestion
-        d = DuplicatesQuestion(self.db, duplicates, self._parent)
+        self.__d_q = d = DuplicatesQuestion(self.db, duplicates, self._parent)
         duplicates = tuple(d.duplicates)
         if duplicates:
             pd = QProgressDialog(_('Adding duplicates...'), '', 0, len(duplicates),

src/calibre/gui2/complete2.py

@@ -14,25 +14,26 @@ from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt, pyqtSignal, QObject,
         QApplication, QListView, QPoint, QModelIndex, QFont, QFontInfo)
 
 from calibre.constants import isosx, get_osx_version
-from calibre.utils.icu import sort_key, primary_startswith, primary_icu_find
+from calibre.utils.icu import sort_key, primary_startswith, primary_contains
 from calibre.gui2 import NONE
 from calibre.gui2.widgets import EnComboBox, LineEditECM
 from calibre.utils.config import tweaks
 
 def containsq(x, prefix):
-    return primary_icu_find(prefix, x)[0] != -1
+    return primary_contains(prefix, x)
 
 class CompleteModel(QAbstractListModel):  # {{{
 
-    def __init__(self, parent=None):
+    def __init__(self, parent=None, sort_func=sort_key):
         QAbstractListModel.__init__(self, parent)
+        self.sort_func = sort_func
         self.all_items = self.current_items = ()
         self.current_prefix = ''
 
     def set_items(self, items):
         items = [unicode(x.strip()) for x in items]
         items = [x for x in items if x]
-        items = tuple(sorted(items, key=sort_key))
+        items = tuple(sorted(items, key=self.sort_func))
         self.all_items = self.current_items = items
         self.current_prefix = ''
         self.reset()
@@ -74,8 +75,9 @@ class Completer(QListView):  # {{{
     item_selected = pyqtSignal(object)
     relayout_needed = pyqtSignal()
 
-    def __init__(self, completer_widget, max_visible_items=7):
+    def __init__(self, completer_widget, max_visible_items=7, sort_func=sort_key):
         QListView.__init__(self)
+        self.disable_popup = False
         self.completer_widget = weakref.ref(completer_widget)
         self.setWindowFlags(Qt.Popup)
         self.max_visible_items = max_visible_items
@@ -84,7 +86,7 @@ class Completer(QListView):  # {{{
         self.setSelectionBehavior(self.SelectRows)
         self.setSelectionMode(self.SingleSelection)
         self.setAlternatingRowColors(True)
-        self.setModel(CompleteModel(self))
+        self.setModel(CompleteModel(self, sort_func=sort_func))
         self.setMouseTracking(True)
         self.entered.connect(self.item_entered)
         self.activated.connect(self.item_chosen)
@@ -132,6 +134,8 @@ class Completer(QListView):  # {{{
                 self.setCurrentIndex(index)
 
     def popup(self, select_first=True):
+        if self.disable_popup:
+            return
         p = self
         m = p.model()
         widget = self.completer_widget()
@@ -253,7 +257,7 @@ class LineEdit(QLineEdit, LineEditECM):
     to complete non multiple fields as well.
     '''
 
-    def __init__(self, parent=None, completer_widget=None):
+    def __init__(self, parent=None, completer_widget=None, sort_func=sort_key):
         QLineEdit.__init__(self, parent)
 
         self.sep = ','
@@ -263,7 +267,7 @@ class LineEdit(QLineEdit, LineEditECM):
         completer_widget = (self if completer_widget is None else
                 completer_widget)
 
-        self.mcompleter = Completer(completer_widget)
+        self.mcompleter = Completer(completer_widget, sort_func=sort_func)
         self.mcompleter.item_selected.connect(self.completion_selected,
                 type=Qt.QueuedConnection)
         self.mcompleter.relayout_needed.connect(self.relayout)
@@ -292,6 +296,13 @@ class LineEdit(QLineEdit, LineEditECM):
             self.mcompleter.model().set_items(items)
         return property(fget=fget, fset=fset)
 
+    @dynamic_property
+    def disable_popup(self):
+        def fget(self):
+            return self.mcompleter.disable_popup
+        def fset(self, val):
+            self.mcompleter.disable_popup = bool(val)
+        return property(fget=fget, fset=fset)
     # }}}
 
     def complete(self, show_all=False, select_first=True):
@@ -303,10 +314,12 @@ class LineEdit(QLineEdit, LineEditECM):
             self.mcompleter.hide()
             return
         self.mcompleter.popup(select_first=select_first)
+        self.setFocus(Qt.OtherFocusReason)
         self.mcompleter.scroll_to(orig)
 
     def relayout(self):
         self.mcompleter.popup()
+        self.setFocus(Qt.OtherFocusReason)
 
     def text_edited(self, *args):
         if self.no_popup:

src/calibre/gui2/convert/djvu_input.py

@@ -1,24 +0,0 @@
-# coding: utf-8
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__   = 'GPL v3'
-__copyright__ = '2011, Anthon van der Neut <A.van.der.Neut@ruamel.eu>'
-
-
-from calibre.gui2.convert.djvu_input_ui import Ui_Form
-from calibre.gui2.convert import Widget
-
-class PluginWidget(Widget, Ui_Form):
-
-    TITLE = _('DJVU Input')
-    HELP = _('Options specific to')+' DJVU '+_('input')
-    COMMIT_NAME = 'djvu_input'
-    ICON = I('mimetypes/djvu.png')
-
-    def __init__(self, parent, get_option, get_help, db=None, book_id=None):
-        Widget.__init__(self, parent,
-            ['use_djvutxt', ])
-        self.db, self.book_id = db, book_id
-        self.initialize_options(get_option, get_help, db, book_id)
-

src/calibre/gui2/convert/djvu_input.ui

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>Form</class>
- <widget class="QWidget" name="Form">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>400</width>
-    <height>300</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>Form</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
-   <item>
-    <widget class="QCheckBox" name="opt_use_djvutxt">
-     <property name="text">
-      <string>Use &amp;djvutxt, if available, for faster processing</string>
-     </property>
-    </widget>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections/>
-</ui>

src/calibre/gui2/cover_flow.py

@@ -21,6 +21,7 @@ pictureflow, pictureflowerror = plugins['pictureflow']
 if pictureflow is not None:
 
     class EmptyImageList(pictureflow.FlowImages):
+
         def __init__(self):
             pictureflow.FlowImages.__init__(self)
 
@@ -108,7 +109,6 @@ if pictureflow is not None:
         def image(self, index):
             return self.model.cover(index)
 
-
     class CoverFlow(pictureflow.PictureFlow):
 
         dc_signal = pyqtSignal()
@@ -125,6 +125,10 @@ if pictureflow is not None:
                     type=Qt.QueuedConnection)
             self.context_menu = None
             self.setContextMenuPolicy(Qt.DefaultContextMenu)
+            try:
+                self.setPreserveAspectRatio(gprefs['cb_preserve_aspect_ratio'])
+            except AttributeError:
+                pass  # source checkout without updated binary
             if hasattr(self, 'setSubtitleFont'):
                 self.setSubtitleFont(QFont(rating_font()))
             if not gprefs['cover_browser_reflections']:
@@ -290,7 +294,6 @@ class CoverFlowMixin(object):
                 self.library_view.setCurrentIndex(idx)
                 self.library_view.scroll_to_row(idx.row())
 
-
     def show_cover_browser(self):
         d = CBDialog(self, self.cover_flow)
         d.addAction(self.cb_splitter.action_toggle)
@@ -313,7 +316,6 @@ class CoverFlowMixin(object):
             self.cb_dialog = None
         self.cb_splitter.button.set_state_to_show()
 
-
     def sync_cf_to_listview(self, current, previous):
         if self.cover_flow_sync_flag and self.cover_flow.isVisible() and \
                 self.cover_flow.currentSlide() != current.row():

src/calibre/gui2/dialogs/tag_editor.py

@@ -9,7 +9,7 @@ from PyQt4.QtGui import QDialog
 from calibre.gui2.dialogs.tag_editor_ui import Ui_TagEditor
 from calibre.gui2 import question_dialog, error_dialog, gprefs
 from calibre.constants import islinux
-from calibre.utils.icu import sort_key, primary_find
+from calibre.utils.icu import sort_key, primary_contains
 
 class TagEditor(QDialog, Ui_TagEditor):
 
@@ -178,7 +178,7 @@ class TagEditor(QDialog, Ui_TagEditor):
         q = icu_lower(unicode(filter_value))
         for i in xrange(collection.count()):  # on every available tag
             item = collection.item(i)
-            item.setHidden(bool(q and primary_find(q, unicode(item.text()))[0] == -1))
+            item.setHidden(bool(q and not primary_contains(q, unicode(item.text()))))
 
     def accept(self):
         self.save_state()

src/calibre/gui2/init.py

@@ -315,7 +315,8 @@ class VLTabs(QTabBar):  # {{{
     def rebuild(self):
         self.currentChanged.disconnect(self.tab_changed)
         db = self.current_db
-        virt_libs = frozenset(db.prefs.get('virtual_libraries', {}))
+        vl_map = db.prefs.get('virtual_libraries', {})
+        virt_libs = frozenset(vl_map)
         hidden = frozenset(db.prefs['virt_libs_hidden'])
         if hidden - virt_libs:
             db.prefs['virt_libs_hidden'] = list(hidden.intersection(virt_libs))
@@ -328,6 +329,9 @@ class VLTabs(QTabBar):  # {{{
         order = {x:i for i, x in enumerate(order)}
         for i, vl in enumerate(sorted(virt_libs, key=lambda x:(order.get(x, 0), sort_key(x)))):
             self.addTab(vl.replace('&', '&&') or _('All books'))
+            sexp = vl_map.get(vl, None)
+            if sexp is not None:
+                self.setTabToolTip(i, _('Search expression for this virtual library:') + '\n\n' + sexp)
             self.setTabData(i, vl)
             if vl == current_lib:
                 current_idx = i

src/calibre/gui2/library/alternate_views.py

@@ -850,4 +850,14 @@ class GridView(QListView):
             except ValueError:
                 pass
 
+    def moveCursor(self, action, modifiers):
+        index = QListView.moveCursor(self, action, modifiers)
+        if action in (QListView.MoveLeft, QListView.MoveRight) and index.isValid():
+            ci = self.currentIndex()
+            if ci.isValid() and index.row() == ci.row():
+                nr = index.row() + (1 if action == QListView.MoveRight else -1)
+                if 0 <= nr < self.model().rowCount(QModelIndex()):
+                    index = self.model().index(nr, 0)
+        return index
+
 # }}}

src/calibre/gui2/main.py

@@ -365,7 +365,7 @@ def cant_start(msg=_('If you are sure it is not running')+', ',
     else:
         where += _('lower right region of the screen.')
     if what is None:
-        if iswindows:
+        if iswindows or islinux:
             what = _('try rebooting your computer.')
         else:
             what = _('try deleting the file')+': '+ gui_socket_address()
@@ -436,7 +436,7 @@ def main(args=sys.argv):
         try:
             listener = Listener(address=gui_socket_address())
         except socket.error:
-            if iswindows:
+            if iswindows or islinux:
                 cant_start()
             if os.path.exists(gui_socket_address()):
                 os.remove(gui_socket_address())

src/calibre/gui2/pictureflow/pictureflow.cpp

@@ -318,6 +318,9 @@ struct SlideInfo
   PFreal cy;
 };
 
+static const QString OFFSET_KEY("offset");
+static const QString WIDTH_KEY("width");
+
 // PicturePlowPrivate {{{
 
 class PictureFlowPrivate
@@ -367,6 +370,7 @@ public:
   QTime  previousPosTimestamp;
   int    pixelDistanceMoved;
   int    pixelsToMovePerSlide;
+  bool   preserveAspectRatio;
   QFont subtitleFont;
 
   void setImages(FlowImages *images);
@@ -421,6 +425,7 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_)
   slideHeight = 200;
   fontSize = 10;
   doReflections = true;
+  preserveAspectRatio = false;
 
   centerIndex = 0;
   queueLength = queueLength_;
@@ -491,9 +496,9 @@ void PictureFlowPrivate::setCurrentSlide(int index)
 {
   animateTimer.stop();
   step = 0;
-  centerIndex = qBound(index, 0, slideImages->count()-1);
+  centerIndex = qBound(0, index, qMax(0, slideImages->count()-1));
   target = centerIndex;
-  slideFrame = ((long long)index) << 16;
+  slideFrame = ((long long)centerIndex) << 16;
   resetSlides();
   triggerRender();
   widget->emitcurrentChanged(centerIndex);
@@ -598,36 +603,53 @@ void PictureFlowPrivate::resetSlides()
   }
 }
 
-static QImage prepareSurface(QImage img, int w, int h, bool doReflections)
+static QImage prepareSurface(QImage srcimg, int w, int h, bool doReflections, bool preserveAspectRatio)
 {
-  img = img.scaled(w, h, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
-
     // slightly larger, to accommodate for the reflection
-  int hs = int(h * REFLECTION_FACTOR);
+    int hs = int(h * REFLECTION_FACTOR), left = 0, top = 0, a = 0, r = 0, g = 0, b = 0, ht, x, y, bpp;
+    QImage img = (preserveAspectRatio) ? QImage(w, h, srcimg.format()) : srcimg.scaled(w, h, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
+    QRgb color;
 
     // offscreen buffer: black is sweet
     QImage result(hs, w, QImage::Format_RGB16);  
     result.fill(0);
 
+    if (preserveAspectRatio) {
+        QImage temp = srcimg.scaled(w, h, Qt::KeepAspectRatio, Qt::SmoothTransformation);
+        img = QImage(w, h, temp.format());  
+        img.fill(0);
+        left = (w - temp.width()) / 2;
+        top = h - temp.height();
+        bpp = img.bytesPerLine() / img.width();
+        x = temp.width() * bpp;
+        result.setText(OFFSET_KEY, QString::number(left));
+        result.setText(WIDTH_KEY, QString::number(temp.width()));
+        for (y = 0; y < temp.height(); y++) {
+            const uchar *src = temp.scanLine(y);
+            uchar *dest = img.scanLine(top + y) + (bpp * left);
+            memcpy(dest, src, x);
+        }
+    }
+
     // transpose the image, this is to speed-up the rendering
     // because we process one column at a time
     // (and much better and faster to work row-wise, i.e in one scanline)
-  for(int x = 0; x < w; x++)
+    for(x = 0; x < w; x++)
-    for(int y = 0; y < h; y++)
+        for(y = 0; y < h; y++)
             result.setPixel(y, x, img.pixel(x, y));
 
     if (doReflections) {
         // create the reflection
-    int ht = hs - h;
+        ht = hs - h;
-    for(int x = 0; x < w; x++)
+        for(x = 0; x < w; x++)
-        for(int y = 0; y < ht; y++)
+            for(y = 0; y < ht; y++)
             {
-        QRgb color = img.pixel(x, img.height()-y-1);
+                color = img.pixel(x, img.height()-y-1);
                 //QRgb565 color = img.scanLine(img.height()-y-1) + x*sizeof(QRgb565); //img.pixel(x, img.height()-y-1);
-        int a = qAlpha(color);
+                a = qAlpha(color);
-        int r = qRed(color)   * a / 256 * (ht - y) / ht * 3/5;
+                r = qRed(color)   * a / 256 * (ht - y) / ht * 3/5;
-        int g = qGreen(color) * a / 256 * (ht - y) / ht * 3/5;
+                g = qGreen(color) * a / 256 * (ht - y) / ht * 3/5;
-        int b = qBlue(color)  * a / 256 * (ht - y) / ht * 3/5;
+                b = qBlue(color)  * a / 256 * (ht - y) / ht * 3/5;
                 result.setPixel(h+y, x, qRgb(r, g, b));
             }
     }
@@ -668,12 +690,12 @@ QImage* PictureFlowPrivate::surface(int slideIndex)
       painter.setBrush(QBrush());
       painter.drawRect(2, 2, slideWidth-3, slideHeight-3);
       painter.end();
-      blankSurface = prepareSurface(blankSurface, slideWidth, slideHeight, doReflections);
+      blankSurface = prepareSurface(blankSurface, slideWidth, slideHeight, doReflections, preserveAspectRatio);
     }
     return &blankSurface;
   }
 
-  surfaceCache.insert(slideIndex, new QImage(prepareSurface(img, slideWidth, slideHeight, doReflections)));
+  surfaceCache.insert(slideIndex, new QImage(prepareSurface(img, slideWidth, slideHeight, doReflections, preserveAspectRatio)));
   return surfaceCache[slideIndex];
 }
 
@@ -874,8 +896,7 @@ QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) {
 // Renders a slide to offscreen buffer. Returns a rect of the rendered area.
 // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent
 // col1 and col2 limit the column for rendering.
-QRect PictureFlowPrivate::renderSlide(const SlideInfo &slide, int alpha, 
+QRect PictureFlowPrivate::renderSlide(const SlideInfo &slide, int alpha, int col1, int col2)
-int col1, int col2)
 {
   QImage* src = surface(slide.slideIndex);
   if(!src)
@@ -913,6 +934,13 @@ int col1, int col2)
 
   bool flag = false;
   rect.setLeft(xi);
+  int img_offset = 0, img_width = 0;
+  bool slide_moving_to_center = false;
+  if (preserveAspectRatio) {
+      img_offset = src->text(OFFSET_KEY).toInt();
+      img_width = src->text(WIDTH_KEY).toInt();
+      slide_moving_to_center = slide.slideIndex == target && target != centerIndex;
+  }
   for(int x = qMax(xi, col1); x <= col2; x++)
   {
     PFreal hity = 0;
@@ -935,6 +963,17 @@ int col1, int col2)
       break;
     if(column < 0)
       continue;
+    if (preserveAspectRatio && !slide_moving_to_center) {
+        // We dont want a black border at the edge of narrow images when the images are in the left or right stacks
+        if (slide.slideIndex < centerIndex) {
+            column = qMin(column + img_offset, sw - 1);
+        } else if (slide.slideIndex == centerIndex) {
+            if (target > centerIndex) column = qMin(column + img_offset, sw - 1);
+            else if (target < centerIndex) column = qMax(column - sw + img_offset + img_width, 0);
+        } else {
+            column = qMax(column - sw + img_offset + img_width, 0);
+        }
+    }
 
     rect.setRight(x);  
     if(!flag)
@@ -1196,6 +1235,17 @@ void PictureFlow::setSlideSize(QSize size)
   d->setSlideSize(size);
 }
 
+bool PictureFlow::preserveAspectRatio() const
+{
+  return d->preserveAspectRatio;
+}
+
+void PictureFlow::setPreserveAspectRatio(bool preserve)
+{
+  d->preserveAspectRatio = preserve;
+  clearCaches();
+}
+
 void PictureFlow::setSubtitleFont(QFont font)
 {
   d->subtitleFont = font;

src/calibre/gui2/pictureflow/pictureflow.h

@@ -93,6 +93,7 @@ Q_OBJECT
   Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide)
   Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize)
   Q_PROPERTY(QFont subtitleFont READ subtitleFont WRITE setSubtitleFont)
+  Q_PROPERTY(bool preserveAspectRatio READ preserveAspectRatio WRITE setPreserveAspectRatio)
 
 public:
   /*!
@@ -121,6 +122,16 @@ public:
   */  
   void setSlideSize(QSize size);
 
+  /*!
+    Returns whether aspect ration is preserved when scaling images
+  */  
+  bool preserveAspectRatio() const;
+
+  /*!
+    Whether to preserve aspect ration when scaling images
+  */  
+  void setPreserveAspectRatio(bool preserve);
+
   /*!
     Turn the reflections on/off.
   */  

src/calibre/gui2/pictureflow/pictureflow.sip

@@ -41,6 +41,10 @@ public :
 
   void setSlideSize(QSize size);
 
+  bool preserveAspectRatio() const;
+
+  void setPreserveAspectRatio(bool preserve);
+
   QFont subtitleFont() const;
 
   void setSubtitleFont(QFont font);

src/calibre/gui2/preferences/look_feel.py

@@ -183,6 +183,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         r('use_roman_numerals_for_series_number', config)
         r('separate_cover_flow', config, restart_required=True)
         r('cb_fullscreen', gprefs)
+        r('cb_preserve_aspect_ratio', gprefs)
 
         choices = [(_('Off'), 'off'), (_('Small'), 'small'),
             (_('Medium'), 'medium'), (_('Large'), 'large')]
@@ -461,6 +462,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         gui.library_view.refresh_book_details()
         if hasattr(gui.cover_flow, 'setShowReflections'):
             gui.cover_flow.setShowReflections(gprefs['cover_browser_reflections'])
+            gui.cover_flow.setPreserveAspectRatio(gprefs['cb_preserve_aspect_ratio'])
         gui.library_view.refresh_row_sizing()
         gui.grid_view.refresh_settings()
 

src/calibre/gui2/preferences/look_feel.ui

@@ -897,7 +897,7 @@ a few top-level elements.</string>
          </property>
         </widget>
        </item>
-       <item row="5" column="0" colspan="2">
+       <item row="6" column="0" colspan="2">
         <spacer name="verticalSpacer_4">
          <property name="orientation">
           <enum>Qt::Vertical</enum>
@@ -913,14 +913,14 @@ a few top-level elements.</string>
        <item row="1" column="1">
         <widget class="QSpinBox" name="opt_cover_flow_queue_length"/>
        </item>
-       <item row="3" column="0" colspan="2">
+       <item row="4" column="0" colspan="2">
         <widget class="QCheckBox" name="opt_cb_fullscreen">
          <property name="text">
           <string>When showing cover browser in separate window, show it &amp;fullscreen</string>
          </property>
         </widget>
        </item>
-       <item row="4" column="0" colspan="2">
+       <item row="5" column="0" colspan="2">
         <widget class="QLabel" name="fs_help_msg">
          <property name="styleSheet">
           <string notr="true">margin-left: 1.5em</string>
@@ -940,6 +940,17 @@ a few top-level elements.</string>
          </property>
         </widget>
        </item>
+       <item row="3" column="0" colspan="2">
+        <widget class="QCheckBox" name="opt_cb_preserve_aspect_ratio">
+         <property name="toolTip">
+          <string>Show covers in their original aspect ratio instead of resizing
+them to all have the same width and height</string>
+         </property>
+         <property name="text">
+          <string>Preserve &amp;aspect ratio of covers displayed in the cover browser</string>
+         </property>
+        </widget>
+       </item>
       </layout>
      </widget>
     </widget>

src/calibre/gui2/tweak_book/__init__.py

@@ -40,6 +40,8 @@ d['remove_existing_links_when_linking_sheets'] = True
 d['charmap_favorites'] = list(map(ord, '\xa0\u2002\u2003\u2009\xad' '‘’“”‹›«»‚„' '—–§¶†‡©®™' '→⇒•·°±−×÷¼½½¾' '…µ¢£€¿¡¨´¸ˆ˜' 'ÀÁÂÃÄÅÆÇÈÉÊË' 'ÌÍÎÏÐÑÒÓÔÕÖØ' 'ŒŠÙÚÛÜÝŸÞßàá' 'âãäåæçèéêëìí' 'îïðñòóôõöøœš' 'ùúûüýÿþªºαΩ∞'))  # noqa
 d['folders_for_types'] = {'style':'styles', 'image':'images', 'font':'fonts', 'audio':'audio', 'video':'video'}
 d['pretty_print_on_open'] = False
+d['disable_completion_popup_for_search'] = False
+d['saved_searches'] = []
 
 del d
 

src/calibre/gui2/tweak_book/boss.py

@@ -7,14 +7,14 @@ __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import tempfile, shutil, sys, os
-from collections import OrderedDict
 from functools import partial, wraps
 
 from PyQt4.Qt import (
-    QObject, QApplication, QDialog, QGridLayout, QLabel, QSize, Qt, QCursor,
+    QObject, QApplication, QDialog, QGridLayout, QLabel, QSize, Qt,
-    QDialogButtonBox, QIcon, QTimer, QPixmap, QTextBrowser, QVBoxLayout, QInputDialog)
+    QDialogButtonBox, QIcon, QTimer, QPixmap, QTextBrowser, QVBoxLayout,
+    QInputDialog)
 
-from calibre import prints, prepare_string_for_xml, isbytestring
+from calibre import prints, isbytestring
 from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
 from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.polish.main import SUPPORTED, tweak_polish
@@ -27,7 +27,6 @@ from calibre.ebooks.oeb.polish.toc import remove_names_from_toc, find_existing_t
 from calibre.ebooks.oeb.polish.utils import link_stylesheets, setup_cssutils_serialization as scs
 from calibre.gui2 import error_dialog, choose_files, question_dialog, info_dialog, choose_save_file
 from calibre.gui2.dialogs.confirm_delete import confirm
-from calibre.gui2.dialogs.message_box import MessageBox
 from calibre.gui2.tweak_book import set_current_container, current_container, tprefs, actions, editors
 from calibre.gui2.tweak_book.undo import GlobalUndoHistory
 from calibre.gui2.tweak_book.file_list import NewFileDialog
@@ -37,7 +36,10 @@ from calibre.gui2.tweak_book.toc import TOCEditor
 from calibre.gui2.tweak_book.editor import editor_from_syntax, syntax_from_mime
 from calibre.gui2.tweak_book.editor.insert_resource import get_resource_data, NewBook
 from calibre.gui2.tweak_book.preferences import Preferences
-from calibre.gui2.tweak_book.widgets import RationalizeFolders, MultiSplit, ImportForeign
+from calibre.gui2.tweak_book.search import validate_search_request, run_search
+from calibre.gui2.tweak_book.widgets import (
+    RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink,
+    InsertSemantics, BusyCursor)
 
 _diff_dialogs = []
 
@@ -57,14 +59,6 @@ def get_container(*args, **kwargs):
 def setup_cssutils_serialization():
     scs(tprefs['editor_tab_stop_width'])
 
-class BusyCursor(object):
-
-    def __enter__(self):
-        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
-
-    def __exit__(self, *args):
-        QApplication.restoreOverrideCursor()
-
 def in_thread_job(func):
     @wraps(func)
     def ans(*args, **kwargs):
@@ -112,6 +106,9 @@ class Boss(QObject):
         self.gui.image_browser.image_activated.connect(self.image_activated)
         self.gui.checkpoints.revert_requested.connect(self.revert_requested)
         self.gui.checkpoints.compare_requested.connect(self.compare_requested)
+        self.gui.saved_searches.run_saved_searches.connect(self.run_saved_searches)
+        self.gui.central.search_panel.save_search.connect(self.save_search)
+        self.gui.central.search_panel.show_saved_searches.connect(self.show_saved_searches)
 
     def preferences(self):
         p = Preferences(self.gui)
@@ -640,9 +637,26 @@ class Boss(QObject):
                         chosen_name = chosen_image_is_external[0]
                     href = current_container().name_to_href(chosen_name, edname)
                     ed.insert_image(href)
+            elif action[0] == 'insert_hyperlink':
+                self.commit_all_editors_to_container()
+                d = InsertLink(current_container(), edname, initial_text=ed.get_smart_selection(), parent=self.gui)
+                if d.exec_() == d.Accepted:
+                    ed.insert_hyperlink(d.href, d.text)
             else:
                 ed.action_triggered(action)
 
+    def set_semantics(self):
+        self.commit_all_editors_to_container()
+        c = current_container()
+        if c.book_type == 'azw3':
+            return error_dialog(self.gui, _('Not supported'), _(
+                'Semantics are not supported for the AZW3 format.'), show=True)
+        d = InsertSemantics(c, parent=self.gui)
+        if d.exec_() == d.Accepted and d.changed_type_map:
+            self.add_savepoint(_('Before: Set Semantics'))
+            d.apply_changes(current_container())
+            self.apply_container_update_to_gui()
+
     def show_find(self):
         self.gui.central.show_find()
         ed = self.gui.central.current_editor
@@ -657,7 +671,7 @@ class Boss(QObject):
         # Ensure the search panel is visible
         sp.setVisible(True)
         ed = self.gui.central.current_editor
-        name = editor = None
+        name = None
         for n, x in editors.iteritems():
             if x is ed:
                 name = n
@@ -666,158 +680,35 @@ class Boss(QObject):
         if overrides:
             state.update(overrides)
         searchable_names = self.gui.file_list.searchable_names
-        where = state['where']
+        if not validate_search_request(name, searchable_names, getattr(ed, 'has_marked_text', False), state, self.gui):
-        err = None
-        if name is None and where in {'current', 'selected-text'}:
-            err = _('No file is being edited.')
-        elif where == 'selected' and not searchable_names['selected']:
-            err = _('No files are selected in the Files Browser')
-        elif where == 'selected-text' and not ed.has_marked_text:
-            err = _('No text is marked. First select some text, and then use'
-                    ' The "Mark selected text" action in the Search menu to mark it.')
-        if not err and not state['find']:
-            err = _('No search query specified')
-        if err:
-            return error_dialog(self.gui, _('Cannot search'), err, show=True)
-        del err
-
-        files = OrderedDict()
-        do_all = state['wrap'] or action in {'replace-all', 'count'}
-        marked = False
-        if where == 'current':
-            editor = ed
-        elif where in {'styles', 'text', 'selected'}:
-            files = searchable_names[where]
-            if name in files:
-                # Start searching in the current editor
-                editor = ed
-                # Re-order the list of other files so that we search in the same
-                # order every time. Depending on direction, search the files
-                # that come after the current file, or before the current file,
-                # first.
-                lfiles = list(files)
-                idx = lfiles.index(name)
-                before, after = lfiles[:idx], lfiles[idx+1:]
-                if state['direction'] == 'up':
-                    lfiles = list(reversed(before))
-                    if do_all:
-                        lfiles += list(reversed(after)) + [name]
-                else:
-                    lfiles = after
-                    if do_all:
-                        lfiles += before + [name]
-                files = OrderedDict((m, files[m]) for m in lfiles)
-        else:
-            editor = ed
-            marked = True
-
-        def no_match():
-            QApplication.restoreOverrideCursor()
-            msg = '<p>' + _('No matches were found for %s') % ('<pre style="font-style:italic">' + prepare_string_for_xml(state['find']) + '</pre>')
-            if not state['wrap']:
-                msg += '<p>' + _('You have turned off search wrapping, so all text might not have been searched.'
-                  ' Try the search again, with wrapping enabled. Wrapping is enabled via the'
-                  ' "Wrap" checkbox at the bottom of the search panel.')
-            return error_dialog(
-                self.gui, _('Not found'), msg, show=True)
-
-        pat = sp.get_regex(state)
-
-        def do_find():
-            if editor is not None:
-                if editor.find(pat, marked=marked, save_match='gui'):
             return
-                if not files:
+
-                    if not state['wrap']:
+        run_search(state, action, ed, name, searchable_names,
-                        return no_match()
+                   self.gui, self.show_editor, self.edit_file, self.show_current_diff, self.add_savepoint, self.rewind_savepoint, self.set_modified)
-                    return editor.find(pat, wrap=True, marked=marked, save_match='gui') or no_match()
+
-            for fname, syntax in files.iteritems():
+    def saved_searches(self):
-                if fname in editors:
+        self.gui.saved_searches.show(), self.gui.saved_searches.raise_()
-                    if not editors[fname].find(pat, complete=True, save_match='gui'):
+
-                        continue
+    def save_search(self):
-                    return self.show_editor(fname)
+        state = self.gui.central.search_panel.state
-                raw = current_container().raw_data(fname)
+        self.show_saved_searches()
-                if pat.search(raw) is not None:
+        self.gui.saved_searches.add_predefined_search(state)
-                    self.edit_file(fname, syntax)
+
-                    if editors[fname].find(pat, complete=True, save_match='gui'):
+    def show_saved_searches(self):
+        self.gui.saved_searches.show(), self.gui.saved_searches.raise_()
+
+    def run_saved_searches(self, searches, action):
+        ed = self.gui.central.current_editor
+        name = None
+        for n, x in editors.iteritems():
+            if x is ed:
+                name = n
+                break
+        searchable_names = self.gui.file_list.searchable_names
+        if not searches or not validate_search_request(name, searchable_names, getattr(ed, 'has_marked_text', False), searches[0], self.gui):
             return
-            return no_match()
+        run_search(searches, action, ed, name, searchable_names,
-
+                   self.gui, self.show_editor, self.edit_file, self.show_current_diff, self.add_savepoint, self.rewind_savepoint, self.set_modified)
-        def no_replace(prefix=''):
-            QApplication.restoreOverrideCursor()
-            if prefix:
-                prefix += ' '
-            error_dialog(
-                self.gui, _('Cannot replace'), prefix + _(
-                'You must first click Find, before trying to replace'), show=True)
-            return False
-
-        def do_replace():
-            if editor is None:
-                return no_replace()
-            if not editor.replace(pat, state['replace'], saved_match='gui'):
-                return no_replace(_(
-                        'Currently selected text does not match the search query.'))
-            return True
-
-        def count_message(action, count, show_diff=False):
-            msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=state['find'], action=action))
-            if show_diff and count > 0:
-                d = MessageBox(MessageBox.INFO, _('Searching done'), prepare_string_for_xml(msg), parent=self.gui, show_copy_button=False)
-                d.diffb = b = d.bb.addButton(_('See what &changed'), d.bb.ActionRole)
-                b.setIcon(QIcon(I('diff.png'))), d.set_details(None), b.clicked.connect(d.accept)
-                b.clicked.connect(partial(self.show_current_diff, allow_revert=True))
-                d.exec_()
-            else:
-                info_dialog(self.gui, _('Searching done'), prepare_string_for_xml(msg), show=True)
-
-        def do_all(replace=True):
-            count = 0
-            if not files and editor is None:
-                return 0
-            lfiles = files or {name:editor.syntax}
-
-            for n, syntax in lfiles.iteritems():
-                if n in editors:
-                    raw = editors[n].get_raw_data()
-                else:
-                    raw = current_container().raw_data(n)
-                if replace:
-                    raw, num = pat.subn(state['replace'], raw)
-                else:
-                    num = len(pat.findall(raw))
-                count += num
-                if replace and num > 0:
-                    if n in editors:
-                        editors[n].replace_data(raw)
-                    else:
-                        with current_container().open(n, 'wb') as f:
-                            f.write(raw.encode('utf-8'))
-            QApplication.restoreOverrideCursor()
-            count_message(_('Replaced') if replace else _('Found'), count, show_diff=replace)
-            return count
-
-        with BusyCursor():
-            if action == 'find':
-                return do_find()
-            if action == 'replace':
-                return do_replace()
-            if action == 'replace-find' and do_replace():
-                return do_find()
-            if action == 'replace-all':
-                if marked:
-                    return count_message(_('Replaced'), editor.all_in_marked(pat, state['replace']))
-                self.add_savepoint(_('Before: Replace all'))
-                count = do_all()
-                if count == 0:
-                    self.rewind_savepoint()
-                else:
-                    self.set_modified()
-                return
-            if action == 'count':
-                if marked:
-                    return count_message(_('Found'), editor.all_in_marked(pat))
-                return do_all(replace=False)
 
     def create_checkpoint(self):
         text, ok = QInputDialog.getText(self.gui, _('Choose name'), _(
@@ -1129,6 +1020,13 @@ class Boss(QObject):
                 _('Editing files of type %s is not supported' % mime), show=True)
         return self.edit_file(name, syntax)
 
+    def quick_open(self):
+        c = current_container()
+        files = [name for name, mime in c.mime_map.iteritems() if c.exists(name) and syntax_from_mime(name, mime) is not None]
+        d = QuickOpen(files, parent=self.gui)
+        if d.exec_() == d.Accepted and d.selected_result is not None:
+            self.edit_file_requested(d.selected_result, None, c.mime_map[d.selected_result])
+
     # Editor basic controls {{{
     def do_editor_undo(self):
         ed = self.gui.central.current_editor

src/calibre/gui2/tweak_book/char_select.py

@@ -21,7 +21,7 @@ from calibre.constants import plugins, cache_dir
 from calibre.gui2 import NONE
 from calibre.gui2.widgets2 import HistoryLineEdit2
 from calibre.gui2.tweak_book import tprefs
-from calibre.gui2.tweak_book.widgets import Dialog
+from calibre.gui2.tweak_book.widgets import Dialog, BusyCursor
 from calibre.utils.icu import safe_chr as chr, icu_unicode_version, character_name_from_code
 
 ROOT = QModelIndex()
@@ -765,7 +765,6 @@ class CharSelect(Dialog):
         self.char_view.setFocus(Qt.OtherFocusReason)
 
     def do_search(self):
-        from calibre.gui2.tweak_book.boss import BusyCursor
         text = unicode(self.search.text()).strip()
         if not text:
             return self.clear_search()

src/calibre/gui2/tweak_book/check.py

@@ -15,6 +15,7 @@ from PyQt4.Qt import (
 from calibre.ebooks.oeb.polish.check.base import WARN, INFO, DEBUG, ERROR, CRITICAL
 from calibre.ebooks.oeb.polish.check.main import run_checks, fix_errors
 from calibre.gui2.tweak_book import tprefs
+from calibre.gui2.tweak_book.widgets import BusyCursor
 
 def icon_for_level(level):
     if level > WARN:
@@ -160,7 +161,6 @@ class Check(QSplitter):
                 template % (err.HELP, ifix, fix_tt, fix_msg, run_tt, run_msg))
 
     def run_checks(self, container):
-        from calibre.gui2.tweak_book.boss import BusyCursor
         with BusyCursor():
             self.show_busy()
             QApplication.processEvents()
@@ -179,7 +179,6 @@ class Check(QSplitter):
             self.clear_help(_('No problems found'))
 
     def fix_errors(self, container, errors):
-        from calibre.gui2.tweak_book.boss import BusyCursor
         with BusyCursor():
             self.show_busy(_('Running fixers, please wait...'))
             QApplication.processEvents()

src/calibre/gui2/tweak_book/editor/smart/__init__.py

@@ -14,3 +14,6 @@ class NullSmarts(object):
     def get_extra_selections(self, editor):
         return ()
 
+    def get_smart_selection(self, editor, update=True):
+        return editor.selected_text
+

src/calibre/gui2/tweak_book/editor/smart/html.py

@@ -12,6 +12,7 @@ from . import NullSmarts
 
 from PyQt4.Qt import QTextEdit
 
+from calibre import prepare_string_for_xml
 from calibre.gui2 import error_dialog
 
 get_offset = itemgetter(0)
@@ -128,6 +129,25 @@ def rename_tag(cursor, opening_tag, closing_tag, new_name, insert=False):
     cursor.insertText(text)
     cursor.endEditBlock()
 
+def ensure_not_within_tag_definition(cursor, forward=True):
+    ''' Ensure the cursor is not inside a tag definition <>. Returns True iff the cursor was moved. '''
+    block, offset = cursor.block(), cursor.positionInBlock()
+    b, boundary = next_tag_boundary(block, offset, forward=False)
+    if b is None:
+        return False
+    if boundary.is_start:
+        # We are inside a tag
+        if forward:
+            block, boundary = next_tag_boundary(block, offset)
+            if block is not None:
+                cursor.setPosition(block.position() + boundary.offset + 1)
+                return True
+        else:
+            cursor.setPosition(b.position() + boundary.offset)
+            return True
+
+    return False
+
 class HTMLSmarts(NullSmarts):
 
     def get_extra_selections(self, editor):
@@ -180,4 +200,35 @@ class HTMLSmarts(NullSmarts):
             return error_dialog(editor, _('No found'), _(
                 'No suitable block level tag was found to rename'), show=True)
 
+    def get_smart_selection(self, editor, update=True):
+        cursor = editor.textCursor()
+        if not cursor.hasSelection():
+            return ''
+        left = min(cursor.anchor(), cursor.position())
+        right = max(cursor.anchor(), cursor.position())
 
+        cursor.setPosition(left)
+        ensure_not_within_tag_definition(cursor)
+        left = cursor.position()
+
+        cursor.setPosition(right)
+        ensure_not_within_tag_definition(cursor, forward=False)
+        right = cursor.position()
+
+        cursor.setPosition(left), cursor.setPosition(right, cursor.KeepAnchor)
+        if update:
+            editor.setTextCursor(cursor)
+        return editor.selected_text_from_cursor(cursor)
+
+    def insert_hyperlink(self, editor, target, text):
+        c = editor.textCursor()
+        if c.hasSelection():
+            c.insertText('')  # delete any existing selected text
+        ensure_not_within_tag_definition(c)
+        c.insertText('<a href="%s">' % prepare_string_for_xml(target, True))
+        p = c.position()
+        c.insertText('</a>')
+        c.setPosition(p)  # ensure cursor is positioned inside the newly created tag
+        if text:
+            c.insertText(text)
+        editor.setTextCursor(c)

src/calibre/gui2/tweak_book/editor/syntax/html.py

@@ -286,7 +286,7 @@ def closing_tag(state, text, i, formats):
 def in_comment(state, text, i, formats):
     ' Comment, processing instruction or doctype '
     end = {state.IN_COMMENT:'-->', state.IN_PI:'?>'}.get(state.parse, '>')
-    pos = text.find(end, i+1)
+    pos = text.find(end, i)
     fmt = formats['comment' if state.parse == state.IN_COMMENT else 'preproc']
     if pos == -1:
         num = len(text) - i
@@ -371,6 +371,8 @@ if __name__ == '__main__':
     launch_editor('''\
 <!DOCTYPE html>
 <html xml:lang="en" lang="en">
+<!--
+-->
     <head>
         <meta charset="utf-8" />
         <title>A title with a tag <span> in it, the tag is treated as normal text</title>

src/calibre/gui2/tweak_book/editor/text.py

@@ -81,8 +81,11 @@ class PlainTextEdit(QPlainTextEdit):
         if hasattr(ans, 'rstrip'):
             ans = ans.rstrip('\0')
         else:  # QString
+            try:
                 while ans[-1] == '\0':
                     ans.chop(1)
+            except IndexError:
+                pass  # ans is an empty string
         return ans
 
     @pyqtSlot()
@@ -101,9 +104,12 @@ class PlainTextEdit(QPlainTextEdit):
         self.copy()
         self.textCursor().removeSelectedText()
 
+    def selected_text_from_cursor(self, cursor):
+        return unicodedata.normalize('NFC', unicode(cursor.selectedText()).replace(PARAGRAPH_SEPARATOR, '\n').rstrip('\0'))
+
     @property
     def selected_text(self):
-        return unicodedata.normalize('NFC', unicode(self.textCursor().selectedText()).replace(PARAGRAPH_SEPARATOR, '\n').rstrip('\0'))
+        return self.selected_text_from_cursor(self.textCursor())
 
     def selection_changed(self):
         # Workaround Qt replacing nbsp with normal spaces on copy
@@ -309,7 +315,7 @@ class TextEdit(PlainTextEdit):
         # Center search result on screen
         self.centerCursor()
         if save_match is not None:
-            self.saved_matches[save_match] = m
+            self.saved_matches[save_match] = (pat, m)
         return True
 
     def all_in_marked(self, pat, template=None):
@@ -366,7 +372,7 @@ class TextEdit(PlainTextEdit):
         # Center search result on screen
         self.centerCursor()
         if save_match is not None:
-            self.saved_matches[save_match] = m
+            self.saved_matches[save_match] = (pat, m)
         return True
 
     def replace(self, pat, template, saved_match='gui'):
@@ -379,8 +385,8 @@ class TextEdit(PlainTextEdit):
             # the saved match matches the currently selected text and
             # use it, if so.
             if saved_match is not None and saved_match in self.saved_matches:
-                saved = self.saved_matches.pop(saved_match)
+                saved_pat, saved = self.saved_matches.pop(saved_match)
-                if saved.group() == raw:
+                if saved_pat == pat and saved.group() == raw:
                     m = saved
         if m is None:
             return False
@@ -602,6 +608,10 @@ class TextEdit(PlainTextEdit):
             c.setPosition(left + len(text), c.KeepAnchor)
         self.setTextCursor(c)
 
+    def insert_hyperlink(self, target, text):
+        if hasattr(self.smarts, 'insert_hyperlink'):
+            self.smarts.insert_hyperlink(self, target, text)
+
     def keyPressEvent(self, ev):
         if ev.key() == Qt.Key_X and ev.modifiers() == Qt.AltModifier:
             if self.replace_possible_unicode_sequence():

src/calibre/gui2/tweak_book/editor/widget.py

@@ -56,6 +56,9 @@ def register_text_editor_actions(reg, palette):
     ac = reg('view-image', _('&Insert image'), ('insert_resource', 'image'), 'insert-image', (), _('Insert an image into the text'))
     ac.setToolTip(_('<h3>Insert image</h3>Insert an image into the text'))
 
+    ac = reg('insert-link', _('Insert &hyperlink'), ('insert_hyperlink',), 'insert-hyperlink', (), _('Insert hyperlink'))
+    ac.setToolTip(_('<h3>Insert hyperlink</h3>Insert a hyperlink into the text'))
+
     for i, name in enumerate(('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p')):
         text = ('&' + name) if name == 'p' else (name[0] + '&' + name[1])
         desc = _('Convert the paragraph to &lt;%s&gt;') % name
@@ -141,6 +144,9 @@ class Editor(QMainWindow):
     def insert_image(self, href):
         self.editor.insert_image(href)
 
+    def insert_hyperlink(self, href, text):
+        self.editor.insert_hyperlink(href, text)
+
     def undo(self):
         self.editor.undo()
 
@@ -151,6 +157,9 @@ class Editor(QMainWindow):
     def selected_text(self):
         return self.editor.selected_text
 
+    def get_smart_selection(self, update=True):
+        return self.editor.smarts.get_smart_selection(self.editor, update=update)
+
     # Search and replace {{{
     def mark_selected_text(self):
         self.editor.mark_selected_text()
@@ -195,6 +204,8 @@ class Editor(QMainWindow):
             b.addAction(actions['pretty-current'])
         if self.syntax in {'html', 'css'}:
             b.addAction(actions['insert-image'])
+        if self.syntax == 'html':
+            b.addAction(actions['insert-hyperlink'])
         if self.syntax == 'html':
             self.format_bar = b = self.addToolBar(_('Format text'))
             for x in ('bold', 'italic', 'underline', 'strikethrough', 'subscript', 'superscript', 'color', 'background-color'):

src/calibre/gui2/tweak_book/matcher.py

@@ -1,146 +0,0 @@
-#!/usr/bin/env python
-# vim:fileencoding=utf-8
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
-
-__license__ = 'GPL v3'
-__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
-
-from unicodedata import normalize
-
-from itertools import izip
-from future_builtins import map
-
-from calibre.constants import plugins
-from calibre.utils.icu import primary_sort_key, find
-
-DEFAULT_LEVEL1 = '/'
-DEFAULT_LEVEL2 = '-_ 0123456789'
-DEFAULT_LEVEL3 = '.'
-
-class Matcher(object):
-
-    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
-        items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items))
-        items = tuple(map(lambda x: x.encode('utf-8'), items))
-        sort_keys = tuple(map(primary_sort_key, items))
-
-        speedup, err = plugins['matcher']
-        if speedup is None:
-            raise RuntimeError('Failed to load the matcher plugin with error: %s' % err)
-        self.m = speedup.Matcher(items, sort_keys, level1.encode('utf-8'), level2.encode('utf-8'), level3.encode('utf-8'))
-
-    def __call__(self, query):
-        query = normalize('NFC', unicode(query)).encode('utf-8')
-        return map(lambda x:x.decode('utf-8'), self.m.get_matches(query))
-
-
-def calc_score_for_char(ctx, prev, current, distance):
-    factor = 1.0
-    ans = ctx.max_score_per_char
-
-    if prev in ctx.level1:
-        factor = 0.9
-    elif prev in ctx.level2 or (icu_lower(prev) == prev and icu_upper(current) == current):
-        factor = 0.8
-    elif prev in ctx.level3:
-        factor = 0.7
-    else:
-        factor = (1.0 / distance) * 0.75
-
-    return ans * factor
-
-def process_item(ctx, haystack, needle):
-    # non-recursive implementation using a stack
-    stack = [(0, 0, 0, 0, [-1]*len(needle))]
-    final_score, final_positions = stack[0][-2:]
-    push, pop = stack.append, stack.pop
-    while stack:
-        hidx, nidx, last_idx, score, positions = pop()
-        key = (hidx, nidx, last_idx)
-        mem = ctx.memory.get(key, None)
-        if mem is None:
-            for i in xrange(nidx, len(needle)):
-                n = needle[i]
-                if (len(haystack) - hidx < len(needle) - i):
-                    score = 0
-                    break
-                pos = find(n, haystack[hidx:])[0] + hidx
-                if pos == -1:
-                    score = 0
-                    break
-
-                distance = pos - last_idx
-                score_for_char = ctx.max_score_per_char if distance <= 1 else calc_score_for_char(ctx, haystack[pos-1], haystack[pos], distance)
-                hidx = pos + 1
-                push((hidx, i, last_idx, score, list(positions)))
-                last_idx = positions[i] = pos
-                score += score_for_char
-            ctx.memory[key] = (score, positions)
-        else:
-            score, positions = mem
-        if score > final_score:
-            final_score = score
-            final_positions = positions
-    return final_score, final_positions
-
-class PyScorer(object):
-    __slots__ = ('level1', 'level2', 'level3', 'max_score_per_char', 'items', 'memory')
-
-    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
-        self.level1, self.level2, self.level3 = level1, level2, level3
-        self.max_score_per_char = 0
-        self.items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items))
-
-    def __call__(self, needle):
-        for item in self.items:
-            self.max_score_per_char = (1.0 / len(item) + 1.0 / len(needle)) / 2.0
-            self.memory = {}
-            yield process_item(self, item, needle)
-
-class CScorer(object):
-
-    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
-        items = tuple(map(lambda x: normalize('NFC', unicode(x)), filter(None, items)))
-
-        speedup, err = plugins['matcher']
-        if speedup is None:
-            raise RuntimeError('Failed to load the matcher plugin with error: %s' % err)
-        self.m = speedup.Matcher(items, unicode(level1), unicode(level2), unicode(level3))
-
-    def __call__(self, query):
-        query = normalize('NFC', unicode(query))
-        scores, positions = self.m.calculate_scores(query)
-        for score, pos in izip(scores, positions):
-            yield score, pos
-
-def test():
-    items = ['m1mn34o/mno']
-    s = PyScorer(items)
-    c = CScorer(items)
-    for q in (s, c):
-        print (q)
-        for item, (score, positions) in izip(items, q('mno')):
-            print (item, score, positions)
-
-def test_mem():
-    from calibre.utils.mem import gc_histogram, diff_hists
-    m = Matcher([])
-    del m
-    def doit(c):
-        m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',])
-        m('one')
-    import gc
-    gc.collect()
-    h1 = gc_histogram()
-    for i in xrange(100):
-        doit(str(i))
-    h2 = gc_histogram()
-    diff_hists(h1, h2)
-
-if __name__ == '__main__':
-    test()
-    # m = Matcher(['image/one.png', 'image/two.gif', 'text/one.html'])
-    # for q in ('one', 'ONE', 'ton', 'imo'):
-    #     print (q, '->', tuple(m(q)))
-    # test_mem()

src/calibre/gui2/tweak_book/preview.py

@@ -416,6 +416,9 @@ class WebView(QWebView):
 
     def contextMenuEvent(self, ev):
         menu = QMenu(self)
+        ca = self.pageAction(QWebPage.Copy)
+        if ca.isEnabled():
+            menu.addAction(ca)
         menu.addAction(actions['reload-preview'])
         menu.addAction(QIcon(I('debug.png')), _('Inspect element'), self.inspect)
         menu.exec_(ev.globalPos())

src/calibre/gui2/tweak_book/save.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import shutil, os
+import shutil, os, errno
 from threading import Thread
 from Queue import LifoQueue, Empty
 
@@ -22,6 +22,20 @@ from calibre.utils.ipc import RC
 def save_container(container, path):
     temp = PersistentTemporaryFile(
         prefix=('_' if iswindows else '.'), suffix=os.path.splitext(path)[1], dir=os.path.dirname(path))
+    if hasattr(os, 'fchmod'):
+        # Ensure file permissions and owner information is preserved
+        fno = temp.fileno()
+        try:
+            st = os.stat(path)
+        except EnvironmentError as err:
+            if err.errno != errno.ENOENT:
+                raise
+            # path may not exist if we are saving a copy, in which case we use
+            # the metadata from the original book
+            st = os.stat(container.path_to_ebook)
+        os.fchmod(fno, st.st_mode)
+        os.fchown(fno, st.st_uid, st.st_gid)
+
     temp.close()
     temp = temp.name
     try:

src/calibre/gui2/tweak_book/search.py

@@ -6,14 +6,26 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
+import json, copy
+from functools import partial
+from collections import OrderedDict
+
 from PyQt4.Qt import (
-    QWidget, QToolBar, Qt, QHBoxLayout, QSize, QIcon, QGridLayout, QLabel,
+    QWidget, QToolBar, Qt, QHBoxLayout, QSize, QIcon, QGridLayout, QLabel, QTimer,
-    QPushButton, pyqtSignal, QComboBox, QCheckBox, QSizePolicy)
+    QPushButton, pyqtSignal, QComboBox, QCheckBox, QSizePolicy, QVBoxLayout,
+    QLineEdit, QToolButton, QListView, QFrame, QApplication, QStyledItemDelegate,
+    QAbstractListModel, QVariant, QFormLayout, QModelIndex, QMenu, QItemSelection)
 
 import regex
 
+from calibre import prepare_string_for_xml
+from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files, choose_save_file
+from calibre.gui2.dialogs.message_box import MessageBox
 from calibre.gui2.widgets2 import HistoryLineEdit2
-from calibre.gui2.tweak_book import tprefs
+from calibre.gui2.tweak_book import tprefs, editors, current_container
+from calibre.gui2.tweak_book.widgets import Dialog, BusyCursor
+
+from calibre.utils.icu import primary_contains
 
 REGEX_FLAGS = regex.VERSION1 | regex.WORD | regex.FULLCASE | regex.MULTILINE | regex.UNICODE
 
@@ -25,6 +37,108 @@ class PushButton(QPushButton):
         QPushButton.__init__(self, text, parent)
         self.clicked.connect(lambda : parent.search_triggered.emit(action))
 
+class HistoryLineEdit(HistoryLineEdit2):
+
+    max_history_items = 100
+    save_search = pyqtSignal()
+    show_saved_searches = pyqtSignal()
+
+    def __init__(self, parent, clear_msg):
+        HistoryLineEdit2.__init__(self, parent)
+        self.disable_popup = tprefs['disable_completion_popup_for_search']
+        self.clear_msg = clear_msg
+
+    def contextMenuEvent(self, event):
+        menu = self.createStandardContextMenu()
+        menu.addSeparator()
+        menu.addAction(self.clear_msg, self.clear_history)
+        menu.addAction((_('Enable completion based on search history') if self.disable_popup else _(
+            'Disable completion based on search history')), self.toggle_popups)
+        menu.addSeparator()
+        menu.addAction(_('Save current search'), self.save_search.emit)
+        menu.addAction(_('Show saved searches'), self.show_saved_searches.emit)
+        menu.exec_(event.globalPos())
+
+    def toggle_popups(self):
+        self.disable_popup = not bool(self.disable_popup)
+        tprefs['disable_completion_popup_for_search'] = self.disable_popup
+
+class WhereBox(QComboBox):
+
+    def __init__(self, parent):
+        QComboBox.__init__(self)
+        self.addItems([_('Current file'), _('All text files'), _('All style files'), _('Selected files'), _('Marked text')])
+        self.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
+            '''
+            Where to search/replace:
+            <dl>
+            <dt><b>Current file</b></dt>
+            <dd>Search only inside the currently opened file</dd>
+            <dt><b>All text files</b></dt>
+            <dd>Search in all text (HTML) files</dd>
+            <dt><b>All style files</b></dt>
+            <dd>Search in all style (CSS) files</dd>
+            <dt><b>Selected files</b></dt>
+            <dd>Search in the files currently selected in the Files Browser</dd>
+            <dt><b>Marked text</b></dt>
+            <dd>Search only within the marked text in the currently opened file. You can mark text using the Search menu.</dd>
+            </dl>'''))
+
+    @dynamic_property
+    def where(self):
+        wm = {0:'current', 1:'text', 2:'styles', 3:'selected', 4:'selected-text'}
+        def fget(self):
+            return wm[self.currentIndex()]
+        def fset(self, val):
+            self.setCurrentIndex({v:k for k, v in wm.iteritems()}[val])
+        return property(fget=fget, fset=fset)
+
+class DirectionBox(QComboBox):
+
+    def __init__(self, parent):
+        QComboBox.__init__(self, parent)
+        self.addItems([_('Down'), _('Up')])
+        self.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
+            '''
+            Direction to search:
+            <dl>
+            <dt><b>Down</b></dt>
+            <dd>Search for the next match from your current position</dd>
+            <dt><b>Up</b></dt>
+            <dd>Search for the previous match from your current position</dd>
+            </dl>'''))
+
+    @dynamic_property
+    def direction(self):
+        def fget(self):
+            return 'down' if self.currentIndex() == 0 else 'up'
+        def fset(self, val):
+            self.setCurrentIndex(1 if val == 'up' else 0)
+        return property(fget=fget, fset=fset)
+
+class ModeBox(QComboBox):
+
+    def __init__(self, parent):
+        QComboBox.__init__(self, parent)
+        self.addItems([_('Normal'), _('Regex')])
+        self.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
+            '''Select how the search expression is interpreted
+            <dl>
+            <dt><b>Normal</b></dt>
+            <dd>The search expression is treated as normal text, calibre will look for the exact text.</dd>
+            <dt><b>Regex</b></dt>
+            <dd>The search expression is interpreted as a regular expression. See the User Manual for more help on using regular expressions.</dd>
+            </dl>'''))
+
+    @dynamic_property
+    def mode(self):
+        def fget(self):
+            return 'normal' if self.currentIndex() == 0 else 'regex'
+        def fset(self, val):
+            self.setCurrentIndex({'regex':1}.get(val, 0))
+        return property(fget=fget, fset=fset)
+
+
 class SearchWidget(QWidget):
 
     DEFAULT_STATE = {
@@ -37,6 +151,8 @@ class SearchWidget(QWidget):
     }
 
     search_triggered = pyqtSignal(object)
+    save_search = pyqtSignal()
+    show_saved_searches = pyqtSignal()
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
@@ -46,7 +162,9 @@ class SearchWidget(QWidget):
 
         self.fl = fl = QLabel(_('&Find:'))
         fl.setAlignment(Qt.AlignRight | Qt.AlignCenter)
-        self.find_text = ft = HistoryLineEdit2(self)
+        self.find_text = ft = HistoryLineEdit(self, _('Clear search history'))
+        ft.save_search.connect(self.save_search)
+        ft.show_saved_searches.connect(self.show_saved_searches)
         ft.initialize('tweak_book_find_edit')
         ft.returnPressed.connect(lambda : self.search_triggered.emit('find'))
         fl.setBuddy(ft)
@@ -55,7 +173,9 @@ class SearchWidget(QWidget):
 
         self.rl = rl = QLabel(_('&Replace:'))
         rl.setAlignment(Qt.AlignRight | Qt.AlignCenter)
-        self.replace_text = rt = HistoryLineEdit2(self)
+        self.replace_text = rt = HistoryLineEdit(self, _('Clear replace history'))
+        rt.save_search.connect(self.save_search)
+        rt.show_saved_searches.connect(self.show_saved_searches)
         rt.initialize('tweak_book_replace_edit')
         rl.setBuddy(rt)
         l.addWidget(rl, 1, 0)
@@ -76,52 +196,17 @@ class SearchWidget(QWidget):
         ml.setAlignment(Qt.AlignRight | Qt.AlignCenter)
         l.addWidget(ml, 2, 0)
         l.addLayout(ol, 2, 1, 1, 3)
-        self.mode_box = mb = QComboBox(self)
+        self.mode_box = mb = ModeBox(self)
         mb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
-        mb.addItems([_('Normal'), _('Regex')])
-        mb.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
-            '''Select how the search expression is interpreted
-            <dl>
-            <dt><b>Normal</b></dt>
-            <dd>The search expression is treated as normal text, calibre will look for the exact text.</dd>
-            <dt><b>Regex</b></dt>
-            <dd>The search expression is interpreted as a regular expression. See the User Manual for more help on using regular expressions.</dd>
-            </dl>'''))
         ml.setBuddy(mb)
         ol.addWidget(mb)
 
-        self.where_box = wb = QComboBox(self)
+        self.where_box = wb = WhereBox(self)
-        wb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
+        wb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed)
-        wb.addItems([_('Current file'), _('All text files'), _('All style files'), _('Selected files'), _('Marked text')])
-        wb.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
-            '''
-            Where to search/replace:
-            <dl>
-            <dt><b>Current file</b></dt>
-            <dd>Search only inside the currently opened file</dd>
-            <dt><b>All text files</b></dt>
-            <dd>Search in all text (HTML) files</dd>
-            <dt><b>All style files</b></dt>
-            <dd>Search in all style (CSS) files</dd>
-            <dt><b>Selected files</b></dt>
-            <dd>Search in the files currently selected in the Files Browser</dd>
-            <dt><b>Marked text</b></dt>
-            <dd>Search only within the marked text in the currently opened file. You can mark text using the Search menu.</dd>
-            </dl>'''))
         ol.addWidget(wb)
 
-        self.direction_box = db = QComboBox(self)
+        self.direction_box = db = DirectionBox(self)
         db.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum)
-        db.addItems([_('Down'), _('Up')])
-        db.setToolTip('<style>dd {margin-bottom: 1.5ex}</style>' + _(
-            '''
-            Direction to search:
-            <dl>
-            <dt><b>Down</b></dt>
-            <dd>Search for the next match from your current position</dd>
-            <dt><b>Up</b></dt>
-            <dd>Search for the previous match from your current position</dd>
-            </dl>'''))
         ol.addWidget(db)
 
         self.cs = cs = QCheckBox(_('&Case sensitive'))
@@ -145,9 +230,9 @@ class SearchWidget(QWidget):
     @dynamic_property
     def mode(self):
         def fget(self):
-            return 'normal' if self.mode_box.currentIndex() == 0 else 'regex'
+            return self.mode_box.mode
         def fset(self, val):
-            self.mode_box.setCurrentIndex({'regex':1}.get(val, 0))
+            self.mode_box.mode = val
             self.da.setVisible(self.mode == 'regex')
         return property(fget=fget, fset=fset)
 
@@ -169,11 +254,10 @@ class SearchWidget(QWidget):
 
     @dynamic_property
     def where(self):
-        wm = {0:'current', 1:'text', 2:'styles', 3:'selected', 4:'selected-text'}
         def fget(self):
-            return wm[self.where_box.currentIndex()]
+            return self.where_box.where
         def fset(self, val):
-            self.where_box.setCurrentIndex({v:k for k, v in wm.iteritems()}[val])
+            self.where_box.where = val
         return property(fget=fget, fset=fset)
 
     @dynamic_property
@@ -187,9 +271,9 @@ class SearchWidget(QWidget):
     @dynamic_property
     def direction(self):
         def fget(self):
-            return 'down' if self.direction_box.currentIndex() == 0 else 'up'
+            return self.direction_box.direction
         def fset(self, val):
-            self.direction_box.setCurrentIndex(1 if val == 'up' else 0)
+            self.direction_box.direction = val
         return property(fget=fget, fset=fset)
 
     @dynamic_property
@@ -236,9 +320,11 @@ class SearchWidget(QWidget):
 
 regex_cache = {}
 
-class SearchPanel(QWidget):
+class SearchPanel(QWidget):  # {{{
 
     search_triggered = pyqtSignal(object)
+    save_search = pyqtSignal()
+    show_saved_searches = pyqtSignal()
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
@@ -257,6 +343,8 @@ class SearchPanel(QWidget):
         l.addWidget(self.widget)
         self.restore_state, self.save_state = self.widget.restore_state, self.widget.save_state
         self.widget.search_triggered.connect(self.search_triggered)
+        self.widget.save_search.connect(self.save_search)
+        self.widget.show_saved_searches.connect(self.show_saved_searches)
         self.pre_fill = self.widget.pre_fill
 
     def hide_panel(self):
@@ -276,7 +364,452 @@ class SearchPanel(QWidget):
     def set_where(self, val):
         self.widget.where = val
 
-    def get_regex(self, state):
+    def keyPressEvent(self, ev):
+        if ev.key() == Qt.Key_Escape:
+            self.hide_panel()
+            ev.accept()
+        else:
+            return QWidget.keyPressEvent(self, ev)
+# }}}
+
+class SearchesModel(QAbstractListModel):
+
+    def __init__(self, parent):
+        QAbstractListModel.__init__(self, parent)
+        self.searches = tprefs['saved_searches']
+        self.filtered_searches = list(xrange(len(self.searches)))
+
+    def rowCount(self, parent=QModelIndex()):
+        return len(self.filtered_searches)
+
+    def data(self, index, role):
+        if role == Qt.DisplayRole:
+            search = self.searches[self.filtered_searches[index.row()]]
+            return QVariant(search['name'])
+        if role == Qt.ToolTipRole:
+            search = self.searches[self.filtered_searches[index.row()]]
+            tt = '\n'.join((search['find'], search['replace']))
+            return QVariant(tt)
+        if role == Qt.UserRole:
+            search = self.searches[self.filtered_searches[index.row()]]
+            return QVariant((self.filtered_searches[index.row()], search))
+        return NONE
+
+    def do_filter(self, text):
+        text = unicode(text)
+        self.filtered_searches = []
+        for i, search in enumerate(self.searches):
+            if primary_contains(text, search['name']):
+                self.filtered_searches.append(i)
+        self.reset()
+
+    def move_entry(self, row, delta):
+        a, b = row, row + delta
+        if 0 <= b < len(self.filtered_searches):
+            ai, bi = self.filtered_searches[a], self.filtered_searches[b]
+            self.searches[ai], self.searches[bi] = self.searches[bi], self.searches[ai]
+            self.dataChanged.emit(self.index(a), self.index(a))
+            self.dataChanged.emit(self.index(b), self.index(b))
+            tprefs['saved_searches'] = self.searches
+
+    def add_searches(self, count=1):
+        self.searches = tprefs['saved_searches']
+        self.filtered_searches.extend(xrange(len(self.searches) - 1, len(self.searches) - 1 - count, -1))
+        self.reset()
+
+    def remove_searches(self, rows):
+        rows = sorted(set(rows), reverse=True)
+        indices = [self.filtered_searches[row] for row in rows]
+        for row in rows:
+            self.beginRemoveRows(QModelIndex(), row, row)
+            del self.filtered_searches[row]
+            self.endRemoveRows()
+        for idx in sorted(indices, reverse=True):
+            del self.searches[idx]
+        tprefs['saved_searches'] = self.searches
+
+class EditSearch(Dialog):  # {{{
+
+    def __init__(self, search=None, search_index=-1, parent=None, state=None):
+        self.search = search or {}
+        self.original_name = self.search.get('name', None)
+        self.search_index = search_index
+        Dialog.__init__(self, _('Edit search'), 'edit-saved-search', parent=parent)
+        if state is not None:
+            self.find.setText(state['find'])
+            self.replace.setText(state['replace'])
+            self.case_sensitive.setChecked(state['case_sensitive'])
+            self.dot_all.setChecked(state['dot_all'])
+            self.mode_box.mode = state.get('mode')
+
+    def sizeHint(self):
+        ans = Dialog.sizeHint(self)
+        ans.setWidth(600)
+        return ans
+
+    def setup_ui(self):
+        self.l = l = QFormLayout(self)
+        self.setLayout(l)
+
+        self.search_name = n = QLineEdit(self.search.get('name', ''), self)
+        n.setPlaceholderText(_('The name with which to save this search'))
+        l.addRow(_('&Name:'), n)
+
+        self.find = f = QLineEdit(self.search.get('find', ''), self)
+        f.setPlaceholderText(_('The expression to search for'))
+        l.addRow(_('&Find:'), f)
+
+        self.replace = r = QLineEdit(self.search.get('replace', ''), self)
+        r.setPlaceholderText(_('The replace expression'))
+        l.addRow(_('&Replace:'), r)
+
+        self.case_sensitive = c = QCheckBox(_('Case sensitive'))
+        c.setChecked(self.search.get('case_sensitive', SearchWidget.DEFAULT_STATE['case_sensitive']))
+        l.addRow(c)
+
+        self.dot_all = d = QCheckBox(_('Dot matches all'))
+        d.setChecked(self.search.get('dot_all', SearchWidget.DEFAULT_STATE['dot_all']))
+        l.addRow(d)
+
+        self.mode_box = m = ModeBox(self)
+        self.mode_box.mode = self.search.get('mode', 'regex')
+        l.addRow(_('&Mode:'), m)
+
+        l.addRow(self.bb)
+
+    def accept(self):
+        searches = tprefs['saved_searches']
+        all_names = {x['name'] for x in searches} - {self.original_name}
+        n = unicode(self.search_name.text()).strip()
+        search = self.search
+        if not n:
+            return error_dialog(self, _('Must specify name'), _(
+                'You must specify a search name'), show=True)
+        if n in all_names:
+            return error_dialog(self, _('Name exists'), _(
+                'Another search with the name %s already exists') % n, show=True)
+        search['name'] = n
+
+        f = unicode(self.find.text())
+        if not f:
+            return error_dialog(self, _('Must specify find'), _(
+                'You must specify a find expression'), show=True)
+        search['find'] = f
+
+        r = unicode(self.replace.text())
+        search['replace'] = r
+
+        search['dot_all'] = bool(self.dot_all.isChecked())
+        search['case_sensitive'] = bool(self.case_sensitive.isChecked())
+        search['mode'] = self.mode_box.mode
+
+        if self.search_index == -1:
+            searches.append(search)
+        else:
+            searches[self.search_index] = search
+        tprefs.set('saved_searches', searches)
+
+        Dialog.accept(self)
+# }}}
+
+class SearchDelegate(QStyledItemDelegate):
+
+    def sizeHint(self, *args):
+        ans = QStyledItemDelegate.sizeHint(self, *args)
+        ans.setHeight(ans.height() + 4)
+        return ans
+
+class SavedSearches(Dialog):
+
+    run_saved_searches = pyqtSignal(object, object)
+
+    def __init__(self, parent=None):
+        Dialog.__init__(self, _('Saved Searches'), 'saved-searches', parent=parent)
+
+    def sizeHint(self):
+        return QSize(800, 675)
+
+    def setup_ui(self):
+        self.l = l = QVBoxLayout(self)
+        self.setLayout(l)
+
+        self.h = h = QHBoxLayout()
+        self.filter_text = ft = QLineEdit(self)
+        ft.textChanged.connect(self.do_filter)
+        ft.setPlaceholderText(_('Filter displayed searches'))
+        h.addWidget(ft)
+        self.cft = cft = QToolButton(self)
+        cft.setToolTip(_('Clear filter')), cft.setIcon(QIcon(I('clear_left.png')))
+        cft.clicked.connect(ft.clear)
+        h.addWidget(cft)
+        l.addLayout(h)
+
+        self.h2 = h = QHBoxLayout()
+        self.searches = searches = QListView(self)
+        searches.doubleClicked.connect(self.edit_search)
+        self.model = SearchesModel(self.searches)
+        self.model.dataChanged.connect(self.show_details)
+        searches.setModel(self.model)
+        searches.selectionModel().currentChanged.connect(self.show_details)
+        searches.setSelectionMode(searches.ExtendedSelection)
+        self.delegate = SearchDelegate(searches)
+        searches.setItemDelegate(self.delegate)
+        searches.setAlternatingRowColors(True)
+        h.addWidget(searches, stretch=10)
+        self.v = v = QVBoxLayout()
+        h.addLayout(v)
+        l.addLayout(h)
+
+        def pb(text, tooltip=None):
+            b = QPushButton(text, self)
+            b.setToolTip(tooltip or '')
+            b.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed)
+            return b
+
+        mulmsg = '\n\n' + _('The entries are tried in order until the first one matches.')
+
+        for text, action, tooltip in [
+                (_('&Find'), 'find', _('Run the search using the selected entries.') + mulmsg),
+                (_('&Replace'), 'replace', _('Run replace using the selected entries.') + mulmsg),
+                (_('Replace a&nd Find'), 'replace-find', _('Run replace and then find using the selected entries.') + mulmsg),
+                (_('Replace &all'), 'replace-all', _('Run Replace All for all selected entries in the order selected')),
+                (_('&Count all'), 'count', _('Run Count All for all selected entries')),
+        ]:
+            b = pb(text, tooltip)
+            v.addWidget(b)
+            b.clicked.connect(partial(self.run_search, action))
+
+        self.d1 = d = QFrame(self)
+        d.setFrameStyle(QFrame.HLine)
+        v.addWidget(d)
+
+        self.h3 = h = QHBoxLayout()
+        self.upb = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-up.png'))), b.setToolTip(_('Move selected entries up'))
+        b.clicked.connect(partial(self.move_entry, -1))
+        self.dnb = b = QToolButton(self)
+        b.setIcon(QIcon(I('arrow-down.png'))), b.setToolTip(_('Move selected entries down'))
+        b.clicked.connect(partial(self.move_entry, 1))
+        h.addWidget(self.upb), h.addWidget(self.dnb)
+        v.addLayout(h)
+
+        self.eb = b = pb(_('&Edit search'), _('Edit the currently selected search'))
+        b.clicked.connect(self.edit_search)
+        v.addWidget(b)
+
+        self.eb = b = pb(_('Re&move search'), _('Remove the currently selected searches'))
+        b.clicked.connect(self.remove_search)
+        v.addWidget(b)
+
+        self.eb = b = pb(_('&Add search'), _('Add a new saved search'))
+        b.clicked.connect(self.add_search)
+        v.addWidget(b)
+
+        self.d2 = d = QFrame(self)
+        d.setFrameStyle(QFrame.HLine)
+        v.addWidget(d)
+
+        self.where_box = wb = WhereBox(self)
+        self.where = SearchWidget.DEFAULT_STATE['where']
+        v.addWidget(wb)
+        self.direction_box = db = DirectionBox(self)
+        self.direction = SearchWidget.DEFAULT_STATE['direction']
+        v.addWidget(db)
+
+        self.wr = wr = QCheckBox(_('&Wrap'))
+        wr.setToolTip('<p>'+_('When searching reaches the end, wrap around to the beginning and continue the search'))
+        self.wr.setChecked(SearchWidget.DEFAULT_STATE['wrap'])
+        v.addWidget(wr)
+
+        self.description = d = QLabel(' \n \n ')
+        d.setTextFormat(Qt.PlainText)
+        l.addWidget(d)
+
+        l.addWidget(self.bb)
+        self.bb.clear()
+        self.bb.addButton(self.bb.Close)
+        self.ib = b = self.bb.addButton(_('&Import'), self.bb.ActionRole)
+        b.clicked.connect(self.import_searches)
+        self.eb = b = self.bb.addButton(_('E&xport'), self.bb.ActionRole)
+        self.em = m = QMenu(_('Export'))
+        m.addAction(_('Export All'), lambda : QTimer.singleShot(0, partial(self.export_searches, all=True)))
+        m.addAction(_('Export Selected'), lambda : QTimer.singleShot(0, partial(self.export_searches, all=False)))
+        b.setMenu(m)
+
+        self.searches.setFocus(Qt.OtherFocusReason)
+
+    @dynamic_property
+    def where(self):
+        def fget(self):
+            return self.where_box.where
+        def fset(self, val):
+            self.where_box.where = val
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def direction(self):
+        def fget(self):
+            return self.direction_box.direction
+        def fset(self, val):
+            self.direction_box.direction = val
+        return property(fget=fget, fset=fset)
+
+    @dynamic_property
+    def wrap(self):
+        def fget(self):
+            return self.wr.isChecked()
+        def fset(self, val):
+            self.wr.setChecked(bool(val))
+        return property(fget=fget, fset=fset)
+
+    def do_filter(self, text):
+        self.model.do_filter(text)
+        self.searches.scrollTo(self.model.index(0))
+
+    def run_search(self, action):
+        searches, seen = [], set()
+        for index in self.searches.selectionModel().selectedIndexes():
+            if index.row() in seen:
+                continue
+            seen.add(index.row())
+            search = SearchWidget.DEFAULT_STATE.copy()
+            del search['mode']
+            search_index, s = index.data(Qt.UserRole).toPyObject()
+            search.update(s)
+            search['wrap'] = self.wrap
+            search['direction'] = self.direction
+            search['where'] = self.where
+            search['mode'] = search.get('mode', 'regex')
+            searches.append(search)
+        if not searches:
+            return
+        self.run_saved_searches.emit(searches, action)
+
+    def move_entry(self, delta):
+        rows = {index.row() for index in self.searches.selectionModel().selectedIndexes()} - {-1}
+        if rows:
+            with tprefs:
+                for row in sorted(rows, reverse=delta > 0):
+                    self.model.move_entry(row, delta)
+            nrow = row + delta
+            index = self.model.index(nrow)
+            if index.isValid():
+                sm = self.searches.selectionModel()
+                sm.setCurrentIndex(index, sm.ClearAndSelect)
+
+    def edit_search(self):
+        index = self.searches.currentIndex()
+        if index.isValid():
+            search_index, search = index.data(Qt.UserRole).toPyObject()
+            d = EditSearch(search=search, search_index=search_index, parent=self)
+            if d.exec_() == d.Accepted:
+                self.model.dataChanged.emit(index, index)
+
+    def remove_search(self):
+        rows = {index.row() for index in self.searches.selectionModel().selectedIndexes()} - {-1}
+        self.model.remove_searches(rows)
+        self.show_details()
+
+    def add_search(self):
+        d = EditSearch(parent=self)
+        self._add_search(d)
+
+    def _add_search(self, d):
+        if d.exec_() == d.Accepted:
+            self.model.add_searches()
+            index = self.model.index(self.model.rowCount() - 1)
+            self.searches.scrollTo(index)
+            sm = self.searches.selectionModel()
+            sm.setCurrentIndex(index, sm.ClearAndSelect)
+            self.show_details()
+
+    def add_predefined_search(self, state):
+        d = EditSearch(parent=self, state=state)
+        self._add_search(d)
+
+    def show_details(self):
+        self.description.setText(' \n \n ')
+        i = self.searches.currentIndex()
+        if i.isValid():
+            search_index, search = i.data(Qt.UserRole).toPyObject()
+            cs = '✓' if search.get('case_sensitive', SearchWidget.DEFAULT_STATE['case_sensitive']) else '✗'
+            da = '✓' if search.get('dot_all', SearchWidget.DEFAULT_STATE['dot_all']) else '✗'
+            if search.get('mode', SearchWidget.DEFAULT_STATE['mode']) == 'regex':
+                ts = _('(Case sensitive: {0} Dot All: {1})').format(cs, da)
+            else:
+                ts = _('(Case sensitive: {0} [Normal search])').format(cs)
+            self.description.setText(_('{2} {3}\nFind: {0}\nReplace: {1}').format(
+                search.get('find', ''), search.get('replace', ''), search.get('name', ''), ts))
+
+    def import_searches(self):
+        path = choose_files(self, 'import_saved_searches', _('Choose file'), filters=[
+            (_('Saved Searches'), ['json'])], all_files=False, select_only_single_file=True)
+        if path:
+            with open(path[0], 'rb') as f:
+                obj = json.loads(f.read())
+            needed_keys = {'name', 'find', 'replace', 'case_sensitive', 'dot_all', 'mode'}
+            def err():
+                error_dialog(self, _('Invalid data'), _(
+                    'The file %s does not contain valid saved searches') % path, show=True)
+            if not isinstance(obj, dict) or not 'version' in obj or not 'searches' in obj or obj['version'] not in (1,):
+                return err()
+            searches = []
+            for item in obj['searches']:
+                if not isinstance(item, dict) or not set(item.iterkeys()).issuperset(needed_keys):
+                    return err
+                searches.append({k:item[k] for k in needed_keys})
+
+            if searches:
+                tprefs['saved_searches'] = tprefs['saved_searches'] + searches
+                count = len(searches)
+                self.model.add_searches(count=count)
+                sm = self.searches.selectionModel()
+                top, bottom = self.model.index(self.model.rowCount() - count), self.model.index(self.model.rowCount() - 1)
+                sm.select(QItemSelection(top, bottom), sm.ClearAndSelect)
+                self.searches.scrollTo(bottom)
+
+    def export_searches(self, all=True):
+        if all:
+            searches = copy.deepcopy(tprefs['saved_searches'])
+            if not searches:
+                return error_dialog(self, _('No searches'), _(
+                    'No searches available to be saved'), show=True)
+        else:
+            searches = []
+            for index in self.searches.selectionModel().selectedIndexes():
+                search = index.data(Qt.UserRole).toPyObject()[-1]
+                searches.append(search.copy())
+            if not searches:
+                return error_dialog(self, _('No searches'), _(
+                    'No searches selected'), show=True)
+        [s.__setitem__('mode', s.get('mode', 'regex')) for s in searches]
+        path = choose_save_file(self, 'export-saved-searches', _('Choose file'), filters=[
+            (_('Saved Searches'), ['json'])], all_files=False)
+        if path:
+            if not path.lower().endswith('.json'):
+                path += '.json'
+            raw = json.dumps({'version':1, 'searches':searches}, ensure_ascii=False, indent=2, sort_keys=True)
+            with open(path, 'wb') as f:
+                f.write(raw.encode('utf-8'))
+
+def validate_search_request(name, searchable_names, has_marked_text, state, gui_parent):
+    err = None
+    where = state['where']
+    if name is None and where in {'current', 'selected-text'}:
+        err = _('No file is being edited.')
+    elif where == 'selected' and not searchable_names['selected']:
+        err = _('No files are selected in the Files Browser')
+    elif where == 'selected-text' and not has_marked_text:
+        err = _('No text is marked. First select some text, and then use'
+                ' The "Mark selected text" action in the Search menu to mark it.')
+    if not err and not state['find']:
+        err = _('No search query specified')
+    if err:
+        error_dialog(gui_parent, _('Cannot search'), err, show=True)
+        return False
+    return True
+
+def get_search_regex(state):
     raw = state['find']
     if state['mode'] != 'regex':
         raw = regex.escape(raw, special_only=True)
@@ -292,10 +825,178 @@ class SearchPanel(QWidget):
         ans = regex_cache[(flags, raw)] = regex.compile(raw, flags=flags)
     return ans
 
-    def keyPressEvent(self, ev):
+def initialize_search_request(state, action, current_editor, current_editor_name, searchable_names):
-        if ev.key() == Qt.Key_Escape:
+    editor = None
-            self.hide_panel()
+    where = state['where']
-            ev.accept()
+    files = OrderedDict()
+    do_all = state['wrap'] or action in {'replace-all', 'count'}
+    marked = False
+    if where == 'current':
+        editor = current_editor
+    elif where in {'styles', 'text', 'selected'}:
+        files = searchable_names[where]
+        if current_editor_name in files:
+            # Start searching in the current editor
+            editor = current_editor
+            # Re-order the list of other files so that we search in the same
+            # order every time. Depending on direction, search the files
+            # that come after the current file, or before the current file,
+            # first.
+            lfiles = list(files)
+            idx = lfiles.index(current_editor_name)
+            before, after = lfiles[:idx], lfiles[idx+1:]
+            if state['direction'] == 'up':
+                lfiles = list(reversed(before))
+                if do_all:
+                    lfiles += list(reversed(after)) + [current_editor_name]
             else:
-            return QWidget.keyPressEvent(self, ev)
+                lfiles = after
+                if do_all:
+                    lfiles += before + [current_editor_name]
+            files = OrderedDict((m, files[m]) for m in lfiles)
+    else:
+        editor = current_editor
+        marked = True
 
+    return editor, where, files, do_all, marked
+
+def run_search(
+    searches, action, current_editor, current_editor_name, searchable_names,
+    gui_parent, show_editor, edit_file, show_current_diff, add_savepoint, rewind_savepoint, set_modified):
+
+    if isinstance(searches, dict):
+        searches = [searches]
+
+    editor, where, files, do_all, marked = initialize_search_request(searches[0], action, current_editor, current_editor_name, searchable_names)
+    wrap = searches[0]['wrap']
+
+    errfind = searches[0]['find']
+    if len(searches) > 1:
+        errfind = _('the selected searches')
+
+    searches = [(get_search_regex(search), search['replace']) for search in searches]
+
+    def no_match():
+        QApplication.restoreOverrideCursor()
+        msg = '<p>' + _('No matches were found for %s') % ('<pre style="font-style:italic">' + prepare_string_for_xml(errfind) + '</pre>')
+        if not wrap:
+            msg += '<p>' + _('You have turned off search wrapping, so all text might not have been searched.'
+                ' Try the search again, with wrapping enabled. Wrapping is enabled via the'
+                ' "Wrap" checkbox at the bottom of the search panel.')
+        return error_dialog(
+            gui_parent, _('Not found'), msg, show=True)
+
+    def do_find():
+        for p, __ in searches:
+            if editor is not None:
+                if editor.find(p, marked=marked, save_match='gui'):
+                    return
+                if wrap and not files and editor.find(p, wrap=True, marked=marked, save_match='gui'):
+                    return
+            for fname, syntax in files.iteritems():
+                ed = editors.get(fname, None)
+                if ed is not None:
+                    if not wrap and ed is editor:
+                        continue
+                    if ed.find(p, complete=True, save_match='gui'):
+                        return show_editor(fname)
+                else:
+                    raw = current_container().raw_data(fname)
+                    if p.search(raw) is not None:
+                        edit_file(fname, syntax)
+                        if editors[fname].find(p, complete=True, save_match='gui'):
+                            return
+        return no_match()
+
+    def no_replace(prefix=''):
+        QApplication.restoreOverrideCursor()
+        if prefix:
+            prefix += ' '
+        error_dialog(
+            gui_parent, _('Cannot replace'), prefix + _(
+            'You must first click Find, before trying to replace'), show=True)
+        return False
+
+    def do_replace():
+        if editor is None:
+            return no_replace()
+        for p, repl in searches:
+            if editor.replace(p, repl, saved_match='gui'):
+                return True
+        return no_replace(_(
+                'Currently selected text does not match the search query.'))
+
+    def count_message(action, count, show_diff=False):
+        msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=errfind, action=action))
+        if show_diff and count > 0:
+            d = MessageBox(MessageBox.INFO, _('Searching done'), prepare_string_for_xml(msg), parent=gui_parent, show_copy_button=False)
+            d.diffb = b = d.bb.addButton(_('See what &changed'), d.bb.ActionRole)
+            b.setIcon(QIcon(I('diff.png'))), d.set_details(None), b.clicked.connect(d.accept)
+            b.clicked.connect(partial(show_current_diff, allow_revert=True))
+            d.exec_()
+        else:
+            info_dialog(gui_parent, _('Searching done'), prepare_string_for_xml(msg), show=True)
+
+    def do_all(replace=True):
+        count = 0
+        if not files and editor is None:
+            return 0
+        lfiles = files or {current_editor_name:editor.syntax}
+        updates = set()
+        raw_data = {}
+        for n, syntax in lfiles.iteritems():
+            if n in editors:
+                raw = editors[n].get_raw_data()
+            else:
+                raw = current_container().raw_data(n)
+            raw_data[n] = raw
+
+        for p, repl in searches:
+            for n, syntax in lfiles.iteritems():
+                raw = raw_data[n]
+                if replace:
+                    raw, num = p.subn(repl, raw)
+                    if num > 0:
+                        updates.add(n)
+                        raw_data[n] = raw
+                else:
+                    num = len(p.findall(raw))
+                count += num
+
+        for n in updates:
+            raw = raw_data[n]
+            if n in editors:
+                editors[n].replace_data(raw)
+            else:
+                with current_container().open(n, 'wb') as f:
+                    f.write(raw.encode('utf-8'))
+        QApplication.restoreOverrideCursor()
+        count_message(_('Replaced') if replace else _('Found'), count, show_diff=replace)
+        return count
+
+    with BusyCursor():
+        if action == 'find':
+            return do_find()
+        if action == 'replace':
+            return do_replace()
+        if action == 'replace-find' and do_replace():
+            return do_find()
+        if action == 'replace-all':
+            if marked:
+                return count_message(_('Replaced'), sum(editor.all_in_marked(p, repl) for p, repl in searches))
+            add_savepoint(_('Before: Replace all'))
+            count = do_all()
+            if count == 0:
+                rewind_savepoint()
+            else:
+                set_modified()
+            return
+        if action == 'count':
+            if marked:
+                return count_message(_('Found'), sum(editor.all_in_marked(p) for p, __ in searches))
+            return do_all(replace=False)
+
+if __name__ == '__main__':
+    app = QApplication([])
+    d = SavedSearches()
+    d.exec_()

src/calibre/gui2/tweak_book/ui.py

@@ -29,6 +29,7 @@ from calibre.gui2.tweak_book.undo import CheckpointView
 from calibre.gui2.tweak_book.preview import Preview
 from calibre.gui2.tweak_book.search import SearchPanel
 from calibre.gui2.tweak_book.check import Check
+from calibre.gui2.tweak_book.search import SavedSearches
 from calibre.gui2.tweak_book.toc import TOCViewer
 from calibre.gui2.tweak_book.char_select import CharSelect
 from calibre.gui2.tweak_book.editor.widget import register_text_editor_actions
@@ -221,6 +222,7 @@ class Main(MainWindow):
         self.setCentralWidget(self.central)
         self.check_book = Check(self)
         self.toc_view = TOCViewer(self)
+        self.saved_searches = SavedSearches(self)
         self.image_browser = InsertImage(self, for_browsing=True)
         self.insert_char = CharSelect(self)
 
@@ -302,6 +304,8 @@ class Main(MainWindow):
         self.action_new_book = reg('book.png', _('Create &new, empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book'))
         self.action_import_book = reg('book.png', _('&Import an HTML or DOCX file as a new book'),
                                       self.boss.import_book, 'import-book', (), _('Import an HTML or DOCX file as a new book'))
+        self.action_quick_edit = reg('modified.png', _('&Quick open a file to edit'), self.boss.quick_open, 'quick-open', ('Ctrl+T'), _(
+            'Quickly open a file from the book to edit it'))
 
         # Editor actions
         group = _('Editor actions')
@@ -341,6 +345,8 @@ class Main(MainWindow):
                                       _('Insert special character'))
         self.action_rationalize_folders = reg('mimetypes/dir.png', _('&Arrange into folders'), self.boss.rationalize_folders, 'rationalize-folders', (),
                                       _('Arrange into folders'))
+        self.action_set_semantics = reg('tags.png', _('Set &Semantics'), self.boss.set_semantics, 'set-semantics', (),
+                                        _('Set Semantics'))
 
         # Polish actions
         group = _('Polish Book')
@@ -389,6 +395,7 @@ class Main(MainWindow):
                                    'count', keys=('Ctrl+N'), description=_('Count number of matches'))
         self.action_mark = reg(None, _('&Mark selected text'), self.boss.mark_selected_text, 'mark-selected-text', ('Ctrl+Shift+M',), _('Mark selected text'))
         self.action_go_to_line = reg(None, _('Go to &line'), self.boss.go_to_line_number, 'go-to-line-number', ('Ctrl+.',), _('Go to line number'))
+        self.action_saved_searches = reg(None, _('Sa&ved searches'), self.boss.saved_searches, 'saved-searches', (), _('Show the saved searches dialog'))
 
         # Check Book actions
         group = _('Check Book')
@@ -430,6 +437,7 @@ class Main(MainWindow):
         f = b.addMenu(_('&File'))
         f.addAction(self.action_new_file)
         f.addAction(self.action_import_files)
+        f.addSeparator()
         f.addAction(self.action_open_book)
         f.addAction(self.action_new_book)
         f.addAction(self.action_import_book)
@@ -455,6 +463,7 @@ class Main(MainWindow):
         e.addAction(self.action_editor_paste)
         e.addAction(self.action_insert_char)
         e.addSeparator()
+        e.addAction(self.action_quick_edit)
         e.addAction(self.action_preferences)
 
         e = b.addMenu(_('&Tools'))
@@ -468,6 +477,7 @@ class Main(MainWindow):
         e.addAction(self.action_fix_html_all)
         e.addAction(self.action_pretty_all)
         e.addAction(self.action_rationalize_folders)
+        e.addAction(self.action_set_semantics)
         e.addAction(self.action_check_book)
 
         e = b.addMenu(_('&View'))
@@ -500,6 +510,8 @@ class Main(MainWindow):
         a(self.action_mark)
         e.addSeparator()
         a(self.action_go_to_line)
+        e.addSeparator()
+        a(self.action_saved_searches)
 
         e = b.addMenu(_('&Help'))
         a = e.addAction

src/calibre/gui2/tweak_book/widgets.py

@@ -6,12 +6,32 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
+import os
+from itertools import izip
+from collections import OrderedDict
+
 from PyQt4.Qt import (
     QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QVBoxLayout,
-    QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt)
+    QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget,
+    QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption,
+    QAbstractListModel, QModelIndex, QVariant, QStyledItemDelegate, QStyle,
+    QListView, QTextDocument, QSize, QComboBox, QFrame, QCursor)
 
-from calibre.gui2 import error_dialog, choose_files, choose_save_file
+from calibre import prepare_string_for_xml
+from calibre.gui2 import error_dialog, choose_files, choose_save_file, NONE, info_dialog
 from calibre.gui2.tweak_book import tprefs
+from calibre.utils.icu import primary_sort_key, sort_key
+from calibre.utils.matcher import get_char, Matcher
+
+ROOT = QModelIndex()
+
+class BusyCursor(object):
+
+    def __enter__(self):
+        QApplication.setOverrideCursor(QCursor(Qt.WaitCursor))
+
+    def __exit__(self, *args):
+        QApplication.restoreOverrideCursor()
 
 class Dialog(QDialog):
 
@@ -222,8 +242,614 @@ class ImportForeign(Dialog):  # {{{
         return src, dest
 # }}}
 
+# Quick Open {{{
+
+def make_highlighted_text(emph, text, positions):
+    positions = sorted(set(positions) - {-1}, reverse=True)
+    text = prepare_string_for_xml(text)
+    for p in positions:
+        ch = get_char(text, p)
+        text = '%s<span style="%s">%s</span>%s' % (text[:p], emph, ch, text[p+len(ch):])
+    return text
+
+
+class Results(QWidget):
+
+    EMPH = "color:magenta; font-weight:bold"
+    MARGIN = 4
+
+    item_selected = pyqtSignal()
+
+    def __init__(self, parent=None):
+        QWidget.__init__(self, parent=parent)
+
+        self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
+        self.results = ()
+        self.current_result = -1
+        self.max_result = -1
+        self.mouse_hover_result = -1
+        self.setMouseTracking(True)
+        self.setFocusPolicy(Qt.NoFocus)
+        self.text_option = to = QTextOption()
+        to.setWrapMode(to.NoWrap)
+        self.divider = QStaticText('\xa0→ \xa0')
+        self.divider.setTextFormat(Qt.PlainText)
+
+    def item_from_y(self, y):
+        if not self.results:
+            return
+        delta = self.results[0][0].size().height() + self.MARGIN
+        maxy = self.height()
+        pos = 0
+        for i, r in enumerate(self.results):
+            bottom = pos + delta
+            if pos <= y < bottom:
+                return i
+                break
+            pos = bottom
+            if pos > min(y, maxy):
+                break
+        return -1
+
+    def mouseMoveEvent(self, ev):
+        y = ev.pos().y()
+        prev = self.mouse_hover_result
+        self.mouse_hover_result = self.item_from_y(y)
+        if prev != self.mouse_hover_result:
+            self.update()
+
+    def mousePressEvent(self, ev):
+        if ev.button() == 1:
+            i = self.item_from_y(ev.pos().y())
+            if i != -1:
+                ev.accept()
+                self.current_result = i
+                self.update()
+                self.item_selected.emit()
+                return
+        return QWidget.mousePressEvent(self, ev)
+
+    def change_current(self, delta=1):
+        if not self.results:
+            return
+        nc = self.current_result + delta
+        if 0 <= nc <= self.max_result:
+            self.current_result = nc
+            self.update()
+
+    def __call__(self, results):
+        if results:
+            self.current_result = 0
+            prefixes = [QStaticText('<b>%s</b>' % os.path.basename(x)) for x in results]
+            [(p.setTextFormat(Qt.RichText), p.setTextOption(self.text_option)) for p in prefixes]
+            self.maxwidth = max([x.size().width() for x in prefixes])
+            self.results = tuple((prefix, self.make_text(text, positions), text)
+                for prefix, (text, positions) in izip(prefixes, results.iteritems()))
+        else:
+            self.results = ()
+            self.current_result = -1
+        self.max_result = min(10, len(self.results) - 1)
+        self.mouse_hover_result = -1
+        self.update()
+
+    def make_text(self, text, positions):
+        text = QStaticText(make_highlighted_text(self.EMPH, text, positions))
+        text.setTextOption(self.text_option)
+        text.setTextFormat(Qt.RichText)
+        return text
+
+    def paintEvent(self, ev):
+        offset = QPoint(0, 0)
+        p = QPainter(self)
+        p.setClipRect(ev.rect())
+        bottom = self.rect().bottom()
+
+        if self.results:
+            for i, (prefix, full, text) in enumerate(self.results):
+                size = prefix.size()
+                if offset.y() + size.height() > bottom:
+                    break
+                self.max_result = i
+                offset.setX(0)
+                if i in (self.current_result, self.mouse_hover_result):
+                    p.save()
+                    if i != self.current_result:
+                        p.setPen(Qt.DotLine)
+                    p.drawLine(offset, QPoint(self.width(), offset.y()))
+                    p.restore()
+                offset.setY(offset.y() + self.MARGIN // 2)
+                p.drawStaticText(offset, prefix)
+                offset.setX(self.maxwidth + 5)
+                p.drawStaticText(offset, self.divider)
+                offset.setX(offset.x() + self.divider.size().width())
+                p.drawStaticText(offset, full)
+                offset.setY(offset.y() + size.height() + self.MARGIN // 2)
+                if i in (self.current_result, self.mouse_hover_result):
+                    offset.setX(0)
+                    p.save()
+                    if i != self.current_result:
+                        p.setPen(Qt.DotLine)
+                    p.drawLine(offset, QPoint(self.width(), offset.y()))
+                    p.restore()
+        else:
+            p.drawText(self.rect(), Qt.AlignCenter, _('No results found'))
+
+        p.end()
+
+    @property
+    def selected_result(self):
+        try:
+            return self.results[self.current_result][-1]
+        except IndexError:
+            pass
+
+class QuickOpen(Dialog):
+
+    def __init__(self, items, parent=None):
+        self.matcher = Matcher(items)
+        self.matches = ()
+        self.selected_result = None
+        Dialog.__init__(self, _('Choose file to edit'), 'quick-open', parent=parent)
+
+    def sizeHint(self):
+        ans = Dialog.sizeHint(self)
+        ans.setWidth(800)
+        ans.setHeight(max(600, ans.height()))
+        return ans
+
+    def setup_ui(self):
+        self.l = l = QVBoxLayout(self)
+        self.setLayout(l)
+
+        self.text = t = QLineEdit(self)
+        t.textEdited.connect(self.update_matches)
+        l.addWidget(t, alignment=Qt.AlignTop)
+
+        example = '<pre>{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg</pre>'.format(
+            '<span style="%s">' % Results.EMPH, '</span>')
+        chars = '<pre style="%s">ics3</pre>' % Results.EMPH
+
+        self.help_label = hl = QLabel(_(
+            '''<p>Quickly choose a file by typing in just a few characters from the file name into the field above.
+        For example, if want to choose the file:
+        {example}
+        Simply type in the characters:
+        {chars}
+        and press Enter.''').format(example=example, chars=chars))
+        hl.setMargin(50), hl.setAlignment(Qt.AlignTop | Qt.AlignHCenter)
+        l.addWidget(hl)
+        self.results = Results(self)
+        self.results.setVisible(False)
+        self.results.item_selected.connect(self.accept)
+        l.addWidget(self.results)
+
+        l.addWidget(self.bb, alignment=Qt.AlignBottom)
+
+    def update_matches(self, text):
+        text = unicode(text).strip()
+        self.help_label.setVisible(False)
+        self.results.setVisible(True)
+        matches = self.matcher(text, limit=100)
+        self.results(matches)
+        self.matches = tuple(matches)
+
+    def keyPressEvent(self, ev):
+        if ev.key() in (Qt.Key_Up, Qt.Key_Down):
+            ev.accept()
+            self.results.change_current(delta=-1 if ev.key() == Qt.Key_Up else 1)
+            return
+        return Dialog.keyPressEvent(self, ev)
+
+    def accept(self):
+        self.selected_result = self.results.selected_result
+        return Dialog.accept(self)
+
+    @classmethod
+    def test(cls):
+        import os
+        from calibre.utils.matcher import get_items_from_dir
+        items = get_items_from_dir(os.getcwdu(), lambda x:not x.endswith('.pyc'))
+        d = cls(items)
+        d.exec_()
+        print (d.selected_result)
+
+# }}}
+
+# Filterable names list {{{
+
+class NamesDelegate(QStyledItemDelegate):
+
+    def sizeHint(self, option, index):
+        ans = QStyledItemDelegate.sizeHint(self, option, index)
+        ans.setHeight(ans.height() + 10)
+        return ans
+
+    def paint(self, painter, option, index):
+        QStyledItemDelegate.paint(self, painter, option, index)
+        text, positions = index.data(Qt.UserRole).toPyObject()
+        self.initStyleOption(option, index)
+        painter.save()
+        painter.setFont(option.font)
+        p = option.palette
+        c = p.HighlightedText if option.state & QStyle.State_Selected else p.Text
+        group = (p.Active if option.state & QStyle.State_Active else p.Inactive)
+        c = p.color(group, c)
+        painter.setClipRect(option.rect)
+        if positions is None or -1 in positions:
+            painter.setPen(c)
+            painter.drawText(option.rect, Qt.AlignLeft | Qt.AlignVCenter | Qt.TextSingleLine, text)
+        else:
+            to = QTextOption()
+            to.setWrapMode(to.NoWrap)
+            to.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
+            positions = sorted(set(positions) - {-1}, reverse=True)
+            text = '<body>%s</body>' % make_highlighted_text(Results.EMPH, text, positions)
+            doc = QTextDocument()
+            c = 'rgb(%d, %d, %d)'%c.getRgb()[:3]
+            doc.setDefaultStyleSheet(' body { color: %s }'%c)
+            doc.setHtml(text)
+            doc.setDefaultFont(option.font)
+            doc.setDocumentMargin(0.0)
+            doc.setDefaultTextOption(to)
+            height = doc.size().height()
+            painter.translate(option.rect.left(), option.rect.top() + (max(0, option.rect.height() - height) // 2))
+            doc.drawContents(painter)
+        painter.restore()
+
+class NamesModel(QAbstractListModel):
+
+    filtered = pyqtSignal(object)
+
+    def __init__(self, names, parent=None):
+        self.items = []
+        QAbstractListModel.__init__(self, parent)
+        self.set_names(names)
+
+    def set_names(self, names):
+        self.names = names
+        self.matcher = Matcher(names)
+        self.filter('')
+
+    def rowCount(self, parent=ROOT):
+        return len(self.items)
+
+    def data(self, index, role):
+        if role == Qt.UserRole:
+            return QVariant(self.items[index.row()])
+        if role == Qt.DisplayRole:
+            return QVariant('\xa0' * 20)
+        return NONE
+
+    def filter(self, query):
+        query = unicode(query or '')
+        if not query:
+            self.items = tuple((text, None) for text in self.names)
+        else:
+            self.items = tuple(self.matcher(query).iteritems())
+        self.reset()
+        self.filtered.emit(not bool(query))
+
+    def find_name(self, name):
+        for i, (text, positions) in enumerate(self.items):
+            if text == name:
+                return i
+
+def create_filterable_names_list(names, filter_text=None, parent=None):
+    nl = QListView(parent)
+    nl.m = m = NamesModel(names, parent=nl)
+    m.filtered.connect(lambda all_items: nl.scrollTo(m.index(0)))
+    nl.setModel(m)
+    nl.d = NamesDelegate(nl)
+    nl.setItemDelegate(nl.d)
+    f = QLineEdit(parent)
+    f.setPlaceholderText(filter_text or '')
+    f.textEdited.connect(m.filter)
+    return nl, f
+
+# }}}
+
+# Insert Link {{{
+class InsertLink(Dialog):
+
+    def __init__(self, container, source_name, initial_text=None, parent=None):
+        self.container = container
+        self.source_name = source_name
+        self.initial_text = initial_text
+        Dialog.__init__(self, _('Insert Hyperlink'), 'insert-hyperlink', parent=parent)
+        self.anchor_cache = {}
+
+    def sizeHint(self):
+        return QSize(800, 600)
+
+    def setup_ui(self):
+        self.l = l = QVBoxLayout(self)
+        self.setLayout(l)
+
+        self.h = h = QHBoxLayout()
+        l.addLayout(h)
+
+        names = [n for n, linear in self.container.spine_names]
+        fn, f = create_filterable_names_list(names, filter_text=_('Filter files'), parent=self)
+        self.file_names, self.file_names_filter = fn, f
+        fn.selectionModel().selectionChanged.connect(self.selected_file_changed)
+        self.fnl = fnl = QVBoxLayout()
+        self.la1 = la = QLabel(_('Choose a &file to link to:'))
+        la.setBuddy(fn)
+        fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn)
+        h.addLayout(fnl), h.setStretch(0, 2)
+
+        fn, f = create_filterable_names_list([], filter_text=_('Filter locations'), parent=self)
+        self.anchor_names, self.anchor_names_filter = fn, f
+        fn.selectionModel().selectionChanged.connect(self.update_target)
+        fn.doubleClicked.connect(self.accept, type=Qt.QueuedConnection)
+        self.anl = fnl = QVBoxLayout()
+        self.la2 = la = QLabel(_('Choose a &location (anchor) in the file:'))
+        la.setBuddy(fn)
+        fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn)
+        h.addLayout(fnl), h.setStretch(1, 1)
+
+        self.tl = tl = QFormLayout()
+        self.target = t = QLineEdit(self)
+        t.setPlaceholderText(_('The destination (href) for the link'))
+        tl.addRow(_('&Target:'), t)
+        l.addLayout(tl)
+
+        self.text_edit = t = QLineEdit(self)
+        la.setBuddy(t)
+        tl.addRow(_('Te&xt:'), t)
+        t.setText(self.initial_text or '')
+        t.setPlaceholderText(_('The (optional) text for the link'))
+
+        l.addWidget(self.bb)
+
+    def selected_file_changed(self, *args):
+        rows = list(self.file_names.selectionModel().selectedRows())
+        if not rows:
+            self.anchor_names.model().set_names([])
+        else:
+            name, positions = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()
+            self.populate_anchors(name)
+
+    def populate_anchors(self, name):
+        if name not in self.anchor_cache:
+            from calibre.ebooks.oeb.base import XHTML_NS
+            root = self.container.parsed(name)
+            self.anchor_cache[name] = sorted(
+                (set(root.xpath('//*/@id')) | set(root.xpath('//h:a/@name', namespaces={'h':XHTML_NS}))) - {''}, key=primary_sort_key)
+        self.anchor_names.model().set_names(self.anchor_cache[name])
+        self.update_target()
+
+    def update_target(self):
+        rows = list(self.file_names.selectionModel().selectedRows())
+        if not rows:
+            return
+        name = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()[0]
+        if name == self.source_name:
+            href = ''
+        else:
+            href = self.container.name_to_href(name, self.source_name)
+        frag = ''
+        rows = list(self.anchor_names.selectionModel().selectedRows())
+        if rows:
+            anchor = self.anchor_names.model().data(rows[0], Qt.UserRole).toPyObject()[0]
+            if anchor:
+                frag = '#' + anchor
+        href += frag
+        self.target.setText(href or '#')
+
+    @property
+    def href(self):
+        return unicode(self.target.text()).strip()
+
+    @property
+    def text(self):
+        return unicode(self.text_edit.text()).strip()
+
+    @classmethod
+    def test(cls):
+        import sys
+        from calibre.ebooks.oeb.polish.container import get_container
+        c = get_container(sys.argv[-1], tweak_mode=True)
+        d = cls(c, next(c.spine_names)[0])
+        if d.exec_() == d.Accepted:
+            print (d.href, d.text)
+
+# }}}
+
+# Insert Semantics {{{
+
+class InsertSemantics(Dialog):
+
+    def __init__(self, container, parent=None):
+        self.container = container
+        self.anchor_cache = {}
+        self.original_type_map = {item.get('type', ''):(container.href_to_name(item.get('href'), container.opf_name), item.get('href', '').partition('#')[-1])
+            for item in container.opf_xpath('//opf:guide/opf:reference[@href and @type]')}
+        self.final_type_map = self.original_type_map.copy()
+        self.create_known_type_map()
+        Dialog.__init__(self, _('Set Semantics'), 'insert-semantics', parent=parent)
+
+    def sizeHint(self):
+        return QSize(800, 600)
+
+    def create_known_type_map(self):
+        _ = lambda x: x
+        self.known_type_map = {
+            'title-page': _('Title Page'),
+            'toc': _('Table of Contents'),
+            'index': _('Index'),
+            'glossary': _('Glossary'),
+            'acknowledgements': _('Acknowledgements'),
+            'bibliography': _('Bibliography'),
+            'colophon': _('Colophon'),
+            'copyright-page': _('Copyright page'),
+            'dedication': _('Dedication'),
+            'epigraph': _('Epigraph'),
+            'foreword': _('Foreword'),
+            'loi': _('List of Illustrations'),
+            'lot': _('List of Tables'),
+            'notes:': _('Notes'),
+            'preface': _('Preface'),
+            'text': _('Text'),
+        }
+        _ = __builtins__['_']
+        type_map_help = {
+            'title-page': _('Page with title, author, publisher, etc.'),
+            'index': _('Back-of-book style index'),
+            'text': _('First "real" page of content'),
+        }
+        t = _
+        all_types = [(k, (('%s (%s)' % (t(v), type_map_help[k])) if k in type_map_help else t(v))) for k, v in self.known_type_map.iteritems()]
+        all_types.sort(key=lambda x: sort_key(x[1]))
+        self.all_types = OrderedDict(all_types)
+
+    def setup_ui(self):
+        self.l = l = QVBoxLayout(self)
+        self.setLayout(l)
+
+        self.tl = tl = QFormLayout()
+        self.semantic_type = QComboBox(self)
+        for key, val in self.all_types.iteritems():
+            self.semantic_type.addItem(val, key)
+        tl.addRow(_('Type of &semantics:'), self.semantic_type)
+        self.target = t = QLineEdit(self)
+        t.setPlaceholderText(_('The destination (href) for the link'))
+        tl.addRow(_('&Target:'), t)
+        l.addLayout(tl)
+
+        self.hline = hl = QFrame(self)
+        hl.setFrameStyle(hl.HLine)
+        l.addWidget(hl)
+
+        self.h = h = QHBoxLayout()
+        l.addLayout(h)
+
+        names = [n for n, linear in self.container.spine_names]
+        fn, f = create_filterable_names_list(names, filter_text=_('Filter files'), parent=self)
+        self.file_names, self.file_names_filter = fn, f
+        fn.selectionModel().selectionChanged.connect(self.selected_file_changed)
+        self.fnl = fnl = QVBoxLayout()
+        self.la1 = la = QLabel(_('Choose a &file:'))
+        la.setBuddy(fn)
+        fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn)
+        h.addLayout(fnl), h.setStretch(0, 2)
+
+        fn, f = create_filterable_names_list([], filter_text=_('Filter locations'), parent=self)
+        self.anchor_names, self.anchor_names_filter = fn, f
+        fn.selectionModel().selectionChanged.connect(self.update_target)
+        fn.doubleClicked.connect(self.accept, type=Qt.QueuedConnection)
+        self.anl = fnl = QVBoxLayout()
+        self.la2 = la = QLabel(_('Choose a &location (anchor) in the file:'))
+        la.setBuddy(fn)
+        fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn)
+        h.addLayout(fnl), h.setStretch(1, 1)
+
+        self.bb.addButton(self.bb.Help)
+        self.bb.helpRequested.connect(self.help_requested)
+        l.addWidget(self.bb)
+        self.semantic_type_changed()
+        self.semantic_type.currentIndexChanged.connect(self.semantic_type_changed)
+        self.target.textChanged.connect(self.target_text_changed)
+
+    def help_requested(self):
+        d = info_dialog(self, _('About semantics'), _(
+            'Semantics refer to additional information about specific locations in the book.'
+            ' For example, you can specify that a particular location is the dedication or the preface'
+            ' or the table of contents and so on.\n\nFirst choose the type of semantic information, then'
+            ' choose a file and optionally a location within the file to point to.\n\nThe'
+            ' semantic information will be written in the <guide> section of the opf file.'))
+        d.resize(d.sizeHint())
+        d.exec_()
+
+    def semantic_type_changed(self):
+        item_type = unicode(self.semantic_type.itemData(self.semantic_type.currentIndex()).toString())
+        name, frag = self.final_type_map.get(item_type, (None, None))
+        self.show_type(name, frag)
+
+    def show_type(self, name, frag):
+        self.file_names_filter.clear(), self.anchor_names_filter.clear()
+        self.file_names.clearSelection(), self.anchor_names.clearSelection()
+        if name is not None:
+            row = self.file_names.model().find_name(name)
+            if row is not None:
+                sm = self.file_names.selectionModel()
+                sm.select(self.file_names.model().index(row), sm.ClearAndSelect)
+                if frag:
+                    row = self.anchor_names.model().find_name(frag)
+                    if row is not None:
+                        sm = self.anchor_names.selectionModel()
+                        sm.select(self.anchor_names.model().index(row), sm.ClearAndSelect)
+        self.target.blockSignals(True)
+        if name is not None:
+            self.target.setText(name + (('#' + frag) if frag else ''))
+        else:
+            self.target.setText('')
+        self.target.blockSignals(False)
+
+    def target_text_changed(self):
+        name, frag = unicode(self.target.text()).partition('#')[::2]
+        item_type = unicode(self.semantic_type.itemData(self.semantic_type.currentIndex()).toString())
+        self.final_type_map[item_type] = (name, frag or None)
+
+    def selected_file_changed(self, *args):
+        rows = list(self.file_names.selectionModel().selectedRows())
+        if not rows:
+            self.anchor_names.model().set_names([])
+        else:
+            name, positions = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()
+            self.populate_anchors(name)
+
+    def populate_anchors(self, name):
+        if name not in self.anchor_cache:
+            from calibre.ebooks.oeb.base import XHTML_NS
+            root = self.container.parsed(name)
+            self.anchor_cache[name] = sorted(
+                (set(root.xpath('//*/@id')) | set(root.xpath('//h:a/@name', namespaces={'h':XHTML_NS}))) - {''}, key=primary_sort_key)
+        self.anchor_names.model().set_names(self.anchor_cache[name])
+        self.update_target()
+
+    def update_target(self):
+        rows = list(self.file_names.selectionModel().selectedRows())
+        if not rows:
+            return
+        name = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()[0]
+        href = name
+        frag = ''
+        rows = list(self.anchor_names.selectionModel().selectedRows())
+        if rows:
+            anchor = self.anchor_names.model().data(rows[0], Qt.UserRole).toPyObject()[0]
+            if anchor:
+                frag = '#' + anchor
+        href += frag
+        self.target.setText(href or '#')
+
+    @property
+    def changed_type_map(self):
+        return {k:v for k, v in self.final_type_map.iteritems() if v != self.original_type_map.get(k, None)}
+
+    def apply_changes(self, container):
+        from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language
+        from calibre.translations.dynamic import translate
+        lang = get_book_language(container)
+        for item_type, (name, frag) in self.changed_type_map.iteritems():
+            title = self.known_type_map[item_type]
+            if lang:
+                title = translate(lang, title)
+            set_guide_item(container, item_type, title, name, frag=frag)
+
+    @classmethod
+    def test(cls):
+        import sys
+        from calibre.ebooks.oeb.polish.container import get_container
+        c = get_container(sys.argv[-1], tweak_mode=True)
+        d = cls(c)
+        if d.exec_() == d.Accepted:
+            import pprint
+            pprint.pprint(d.changed_type_map)
+            d.apply_changes(d.container)
+
+# }}}
+
 if __name__ == '__main__':
     app = QApplication([])
-    d = ImportForeign()
+    InsertSemantics.test()
-    d.exec_()
-    print (d.data)

src/calibre/gui2/viewer/config.py

@@ -35,6 +35,10 @@ def config(defaults=None):
         help=_("Set the maximum width that the book's text and pictures will take"
         " when in fullscreen mode. This allows you to read the book text"
         " without it becoming too wide."))
+    c.add_opt('max_fs_height', default=-1,
+        help=_("Set the maximum height that the book's text and pictures will take"
+        " when in fullscreen mode. This allows you to read the book text"
+        " without it becoming too tall. Note that this setting only takes effect in paged mode (which is the default mode)."))
     c.add_opt('fit_images', default=True,
             help=_('Resize images larger than the viewer window to fit inside it'))
     c.add_opt('hyphenate', default=False, help=_('Hyphenate text'))
@@ -211,6 +215,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
                 {'serif':0, 'sans':1, 'mono':2}[opts.standard_font])
         self.css.setPlainText(opts.user_css)
         self.max_fs_width.setValue(opts.max_fs_width)
+        self.max_fs_height.setValue(opts.max_fs_height)
         pats, names = self.hyphenate_pats, self.hyphenate_names
         try:
             idx = pats.index(opts.hyphenate_default_lang)
@@ -287,6 +292,10 @@ class ConfigDialog(QDialog, Ui_Dialog):
         c.set('remember_window_size', self.opt_remember_window_size.isChecked())
         c.set('fit_images', self.opt_fit_images.isChecked())
         c.set('max_fs_width', int(self.max_fs_width.value()))
+        max_fs_height = self.max_fs_height.value()
+        if max_fs_height <= self.max_fs_height.minimum():
+            max_fs_height = -1
+        c.set('max_fs_height', max_fs_height)
         c.set('hyphenate', self.hyphenate.isChecked())
         c.set('remember_current_page', self.opt_remember_current_page.isChecked())
         c.set('wheel_flips_pages', self.opt_wheel_flips_pages.isChecked())

src/calibre/gui2/viewer/config.ui

@@ -60,7 +60,7 @@ QToolBox::tab:hover {
 }</string>
          </property>
          <property name="currentIndex">
-          <number>0</number>
+          <number>2</number>
          </property>
          <widget class="QWidget" name="page">
           <property name="geometry">
@@ -404,41 +404,67 @@ QToolBox::tab:hover {
              </property>
             </widget>
            </item>
-           <item row="1" column="0">
+           <item row="6" column="0" colspan="2">
-            <widget class="QCheckBox" name="opt_fullscreen_clock">
-             <property name="text">
-              <string>Show &amp;clock in full screen mode</string>
-             </property>
-            </widget>
-           </item>
-           <item row="5" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_fullscreen_pos">
              <property name="text">
               <string>Show reading &amp;position in full screen mode</string>
              </property>
             </widget>
            </item>
-           <item row="4" column="0" colspan="2">
+           <item row="5" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_fullscreen_scrollbar">
              <property name="text">
               <string>Show &amp;scrollbar in full screen mode</string>
              </property>
             </widget>
            </item>
-           <item row="3" column="0" colspan="2">
+           <item row="4" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_start_in_fullscreen">
              <property name="text">
               <string>&amp;Start viewer in full screen mode</string>
              </property>
             </widget>
            </item>
-           <item row="2" column="0" colspan="2">
+           <item row="3" column="0" colspan="2">
             <widget class="QCheckBox" name="opt_show_fullscreen_help">
              <property name="text">
               <string>Show &amp;help message when starting full screen mode</string>
              </property>
             </widget>
            </item>
+           <item row="1" column="0">
+            <widget class="QLabel" name="label_24">
+             <property name="text">
+              <string>Maximum text height in fullscreen (paged mode):</string>
+             </property>
+            </widget>
+           </item>
+           <item row="2" column="0" colspan="2">
+            <widget class="QCheckBox" name="opt_fullscreen_clock">
+             <property name="text">
+              <string>Show &amp;clock in full screen mode</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="1">
+            <widget class="QSpinBox" name="max_fs_height">
+             <property name="specialValueText">
+              <string>Disabled</string>
+             </property>
+             <property name="suffix">
+              <string> px</string>
+             </property>
+             <property name="minimum">
+              <number>100</number>
+             </property>
+             <property name="maximum">
+              <number>10000</number>
+             </property>
+             <property name="singleStep">
+              <number>25</number>
+             </property>
+            </widget>
+           </item>
           </layout>
          </widget>
          <widget class="QWidget" name="page_6">

src/calibre/gui2/viewer/documentview.py

@@ -160,6 +160,7 @@ class Document(QWebPage):  # {{{
         screen_width = QApplication.desktop().screenGeometry().width()
         # Leave some space for the scrollbar and some border
         self.max_fs_width = min(opts.max_fs_width, screen_width-50)
+        self.max_fs_height = opts.max_fs_height
         self.fullscreen_clock = opts.fullscreen_clock
         self.fullscreen_scrollbar = opts.fullscreen_scrollbar
         self.fullscreen_pos = opts.fullscreen_pos
@@ -280,11 +281,16 @@ class Document(QWebPage):  # {{{
             ))
         force_fullscreen_layout = bool(getattr(last_loaded_path,
                                                'is_single_page', False))
-        f = 'true' if force_fullscreen_layout else 'false'
+        self.update_contents_size_for_paged_mode(force_fullscreen_layout)
-        side_margin = self.javascript('window.paged_display.layout(%s)'%f, typ=int)
+
+    def update_contents_size_for_paged_mode(self, force_fullscreen_layout=None):
         # Setup the contents size to ensure that there is a right most margin.
         # Without this WebKit renders the final column with no margin, as the
         # columns extend beyond the boundaries (and margin) of body
+        if force_fullscreen_layout is None:
+            force_fullscreen_layout = self.javascript('window.paged_display.is_full_screen_layout', typ=bool)
+        f = 'true' if force_fullscreen_layout else 'false'
+        side_margin = self.javascript('window.paged_display.layout(%s)'%f, typ=int)
         mf = self.mainFrame()
         sz = mf.contentsSize()
         scroll_width = self.javascript('document.body.scrollWidth', int)
@@ -310,7 +316,7 @@ class Document(QWebPage):  # {{{
 
     def switch_to_fullscreen_mode(self):
         self.in_fullscreen_mode = True
-        self.javascript('full_screen.on(%d, %s)'%(self.max_fs_width,
+        self.javascript('full_screen.on(%d, %d, %s)'%(self.max_fs_width, self.max_fs_height,
             'true' if self.in_paged_mode else 'false'))
 
     def switch_to_window_mode(self):
@@ -353,6 +359,8 @@ class Document(QWebPage):  # {{{
             return ans[0] if ans[1] else 0.0
         if typ == 'string':
             return unicode(ans.toString())
+        if typ in {bool, 'bool'}:
+            return ans.toBool()
         return ans
 
     def javaScriptConsoleMessage(self, msg, lineno, msgid):
@@ -1103,7 +1111,11 @@ class DocumentView(QWebView):  # {{{
         def fget(self):
             return self.zoomFactor()
         def fset(self, val):
+            oval = self.zoomFactor()
             self.setZoomFactor(val)
+            if val != oval:
+                if self.document.in_paged_mode:
+                    self.document.update_contents_size_for_paged_mode()
                 self.magnification_changed.emit(val)
         return property(fget=fget, fset=fset)
 

src/calibre/gui2/viewer/main.py

@@ -1119,7 +1119,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                 event.accept()
                 return
             if self.isFullScreen():
-                self.toggle_fullscreen()
+                self.action_full_screen.trigger()
                 event.accept()
                 return
         try:

src/calibre/gui2/widgets2.py

@@ -11,6 +11,11 @@ from calibre.gui2.widgets import history
 
 class HistoryLineEdit2(LineEdit):
 
+    max_history_items = None
+
+    def __init__(self, parent=None, completer_widget=None, sort_func=lambda x:None):
+        LineEdit.__init__(self, parent=parent, completer_widget=completer_widget, sort_func=sort_func)
+
     @property
     def store_name(self):
         return 'lineedit_history_'+self._name
@@ -31,6 +36,13 @@ class HistoryLineEdit2(LineEdit):
             except ValueError:
                 pass
             self.history.insert(0, ct)
+            if self.max_history_items is not None:
+                del self.history[self.max_history_items:]
+            history.set(self.store_name, self.history)
+            self.update_items_cache(self.history)
+
+    def clear_history(self):
+        self.history = []
         history.set(self.store_name, self.history)
         self.update_items_cache(self.history)
 

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -584,10 +584,11 @@ class CatalogBuilder(object):
                 if field_contents == '':
                     field_contents = None
 
-                if (self.db.metadata_for_field(rule['field'])['datatype'] == 'bool' and
+                # Handle condition where bools_are_tristate is False,
+                # field is a bool and contents is None, which is displayed as No
+                if (not self.db.prefs.get('bools_are_tristate') and
+                    self.db.metadata_for_field(rule['field'])['datatype'] == 'bool' and
                     field_contents is None):
-                    # Handle condition where field is a bool and contents is None,
-                    # which is displayed as No
                     field_contents = _('False')
 
                 if field_contents is not None:
@@ -1021,8 +1022,11 @@ class CatalogBuilder(object):
         data = self.plugin.search_sort_db(self.db, self.opts)
         data = self.process_exclusions(data)
 
-        if self.prefix_rules and self.DEBUG:
+        if self.DEBUG:
-            self.opts.log.info(" Added prefixes:")
+            if self.prefix_rules:
+                self.opts.log.info(" Added prefixes (bools_are_tristate: {0}):".format(self.db.prefs.get('bools_are_tristate')))
+            else:
+                self.opts.log.info(" No added prefixes")
 
         # Populate this_title{} from data[{},{}]
         titles = []

src/calibre/library/cli.py

@@ -19,6 +19,7 @@ from calibre.ebooks.metadata.meta import get_metadata
 from calibre.ebooks.metadata.book.base import field_from_string
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.date import isoformat
+from calibre.utils.localization import canonicalize_lang
 
 FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
     'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
@@ -229,7 +230,7 @@ class DevNull(object):
 NULL = DevNull()
 
 def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
-        oauthors, oisbn, otags, oseries, oseries_index, ocover):
+        oauthors, oisbn, otags, oseries, oseries_index, ocover, olanguages):
     orig = sys.stdout
     #sys.stdout = NULL
     try:
@@ -256,7 +257,7 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates, otitle,
                 mi.title = os.path.splitext(os.path.basename(book))[0]
             if not mi.authors:
                 mi.authors = [_('Unknown')]
-            for x in ('title', 'authors', 'isbn', 'tags', 'series'):
+            for x in ('title', 'authors', 'isbn', 'tags', 'series', 'languages'):
                 val = locals()['o'+x]
                 if val:
                     setattr(mi, x, val)
@@ -354,10 +355,12 @@ the directory related options below.
             help=_('Set the series number of the added book(s)'))
     parser.add_option('-c', '--cover', default=None,
             help=_('Path to the cover to use for the added book'))
+    parser.add_option('-l', '--languages', default=None,
+            help=_('A comma separated list of languages (best to use ISO639 language codes, though some language names may also be recognized)'))
 
     return parser
 
-def do_add_empty(db, title, authors, isbn, tags, series, series_index, cover):
+def do_add_empty(db, title, authors, isbn, tags, series, series_index, cover, languages):
     from calibre.ebooks.metadata import MetaInformation
     mi = MetaInformation(None)
     if title is not None:
@@ -372,6 +375,8 @@ def do_add_empty(db, title, authors, isbn, tags, series, series_index, cover):
         mi.series, mi.series_index = series, series_index
     if cover:
         mi.cover = cover
+    if languages:
+        mi.languages = languages
     book_id = db.import_book(mi, [])
     write_dirtied(db)
     prints(_('Added book ids: %s')%book_id)
@@ -383,9 +388,11 @@ def command_add(args, dbpath):
     opts, args = parser.parse_args(sys.argv[:1] + args)
     aut = string_to_authors(opts.authors) if opts.authors else []
     tags = [x.strip() for x in opts.tags.split(',')] if opts.tags else []
+    lcodes = [canonicalize_lang(x) for x in (opts.languages or '').split(',')]
+    lcodes = [x for x in lcodes if x]
     if opts.empty:
         do_add_empty(get_db(dbpath, opts), opts.title, aut, opts.isbn, tags,
-                opts.series, opts.series_index, opts.cover)
+                opts.series, opts.series_index, opts.cover, lcodes)
         return 0
     if len(args) < 2:
         parser.print_help()
@@ -394,7 +401,7 @@ def command_add(args, dbpath):
         return 1
     do_add(get_db(dbpath, opts), args[1:], opts.one_book_per_directory,
             opts.recurse, opts.duplicates, opts.title, aut, opts.isbn,
-            tags, opts.series, opts.series_index, opts.cover)
+            tags, opts.series, opts.series_index, opts.cover, lcodes)
     return 0
 
 def do_remove(db, ids):

src/calibre/library/server/base.py

@@ -47,6 +47,10 @@ class DispatchController(object):  # {{{
         aw = kwargs.pop('android_workaround', False)
         if route != '/':
             route = self.prefix + route
+        if isinstance(route, unicode):
+            # Apparently the routes package chokes on unicode routes, see
+            # http://www.mobileread.com/forums/showthread.php?t=235366
+            route = route.encode('utf-8')
         elif self.prefix:
             self.dispatcher.connect(name+'prefix_extra', self.prefix, self,
                     **kwargs)

src/calibre/test_build.py

@@ -113,10 +113,9 @@ def test_ssl():
     print ('SSL OK!')
 
 def test_icu():
-    from calibre.utils.icu import _icu_not_ok, test_roundtrip
+    print ('Testing ICU')
-    if _icu_not_ok:
+    from calibre.utils.icu_test import test_build
-        raise RuntimeError('ICU module not loaded/valid')
+    test_build()
-    test_roundtrip()
     print ('ICU OK!')
 
 def test_wpd():

src/calibre/utils/config.py

@@ -204,7 +204,7 @@ class DynamicConfig(dict):
 
     def decouple(self, prefix):
         self.file_path = os.path.join(os.path.dirname(self.file_path), prefix + os.path.basename(self.file_path))
-        self.refresh(clear_current=False)
+        self.refresh()
 
     def refresh(self, clear_current=True):
         d = {}
@@ -287,7 +287,7 @@ class XMLConfig(dict):
 
     def decouple(self, prefix):
         self.file_path = os.path.join(os.path.dirname(self.file_path), prefix + os.path.basename(self.file_path))
-        self.refresh(clear_current=False)
+        self.refresh()
 
     def refresh(self, clear_current=True):
         d = {}

src/calibre/utils/icu.c

@@ -1,5 +1,9 @@
 #include "icu_calibre_utils.h"
 
+#define UPPER_CASE 0
+#define LOWER_CASE 1
+#define TITLE_CASE 2
+
 static PyObject* uchar_to_unicode(const UChar *src, int32_t len) {
     wchar_t *buf = NULL;
     PyObject *ans = NULL;
@@ -66,20 +70,16 @@ icu_Collator_display_name(icu_Collator *self, void *closure) {
     const char *loc = NULL;
     UErrorCode status = U_ZERO_ERROR;
     UChar dname[400];
-    char buf[100];
+    int32_t sz = 0;
 
     loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
-    if (loc == NULL || U_FAILURE(status)) {
+    if (loc == NULL) {
         PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
     }
-    ucol_getDisplayName(loc, "en", dname, 100, &status);
+    sz = ucol_getDisplayName(loc, "en", dname, sizeof(dname), &status);
-    if (U_FAILURE(status)) return PyErr_NoMemory();
+    if (U_FAILURE(status)) {PyErr_SetString(PyExc_ValueError, u_errorName(status)); return NULL; }
 
-    u_strToUTF8(buf, 100, NULL, dname, -1, &status);
+    return icu_to_python(dname, sz);
-    if (U_FAILURE(status)) {
-        PyErr_SetString(PyExc_Exception, "Failed to convert dname to UTF-8"); return NULL;
-    }
-    return Py_BuildValue("s", buf);
 }
 
 // }}}
@@ -131,50 +131,38 @@ icu_Collator_actual_locale(icu_Collator *self, void *closure) {
 
 // }}}
 
+// Collator.capsule {{{
+static PyObject *
+icu_Collator_capsule(icu_Collator *self, void *closure) {
+    return PyCapsule_New(self->collator, NULL, NULL);
+} // }}}
+
 // Collator.sort_key {{{
 static PyObject *
 icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) {
-    char *input;
+    int32_t sz = 0, key_size = 0, bsz = 0;
-    int32_t sz;
+    UChar *buf = NULL;
-    UChar *buf;
+    uint8_t *buf2 = NULL;
-    uint8_t *buf2;
+    PyObject *ans = NULL, *input = NULL;
-    PyObject *ans;
-    int32_t key_size;
-    UErrorCode status = U_ZERO_ERROR;
   
-    if (!PyArg_ParseTuple(args, "es", "UTF-8", &input)) return NULL;
+    if (!PyArg_ParseTuple(args, "O", &input)) return NULL;
+    buf = python_to_icu(input, &sz, 1);
+    if (buf == NULL) return NULL;
 
-    sz = (int32_t)strlen(input);
+    bsz = 7 * sz + 1;
-
+    buf2 = (uint8_t*)calloc(bsz, sizeof(uint8_t));
-    buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
+    if (buf2 == NULL) { PyErr_NoMemory(); goto end; }
-
+    key_size = ucol_getSortKey(self->collator, buf, sz, buf2, bsz);
-    if (buf == NULL) return PyErr_NoMemory();
+    if (key_size > bsz) {
-
+        buf2 = realloc(buf2, (key_size + 1) * sizeof(uint8_t));
-    u_strFromUTF8(buf, sz*4 + 1, &key_size, input, sz, &status);
+        if (buf2 == NULL) { PyErr_NoMemory(); goto end; }
-    PyMem_Free(input);
+        key_size = ucol_getSortKey(self->collator, buf, sz, buf2, key_size + 1);
-
-    if (U_SUCCESS(status)) {
-        buf2 = (uint8_t*)calloc(7*sz+1, sizeof(uint8_t));
-        if (buf2 == NULL) return PyErr_NoMemory();
-
-        key_size = ucol_getSortKey(self->collator, buf, -1, buf2, 7*sz+1);
-
-        if (key_size == 0) {
-            ans = PyBytes_FromString("");
-        } else {
-            if (key_size >= 7*sz+1) {
-                free(buf2);
-                buf2 = (uint8_t*)calloc(key_size+1, sizeof(uint8_t));
-                if (buf2 == NULL) return PyErr_NoMemory();
-                ucol_getSortKey(self->collator, buf, -1, buf2, key_size+1);
     }
-            ans = PyBytes_FromString((char *)buf2);
+    ans = PyBytes_FromStringAndSize((char*)buf2, key_size);
-        }
-        free(buf2);
-    } else ans = PyBytes_FromString("");
 
-    free(buf);
+end:
-    if (ans == NULL) return PyErr_NoMemory();
+    if (buf != NULL) free(buf);
+    if (buf2 != NULL) free(buf2);
 
     return ans;
 } // }}}
@@ -182,86 +170,106 @@ icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) {
 // Collator.strcmp {{{
 static PyObject *
 icu_Collator_strcmp(icu_Collator *self, PyObject *args, PyObject *kwargs) {
-    char *a_, *b_;
+    PyObject *a_ = NULL, *b_ = NULL;
-    int32_t asz, bsz;
+    int32_t asz = 0, bsz = 0;
-    UChar *a, *b;
+    UChar *a = NULL, *b = NULL;
-    UErrorCode status = U_ZERO_ERROR;
     UCollationResult res = UCOL_EQUAL;
   
-    if (!PyArg_ParseTuple(args, "eses", "UTF-8", &a_, "UTF-8", &b_)) return NULL;
+    if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL;
 
-    asz = (int32_t)strlen(a_); bsz = (int32_t)strlen(b_);
+    a = python_to_icu(a_, &asz, 1);
+    if (a == NULL) goto end;
+    b = python_to_icu(b_, &bsz, 1);
+    if (b == NULL) goto end;
+    res = ucol_strcoll(self->collator, a, asz, b, bsz);
+end:
+    if (a != NULL) free(a); if (b != NULL) free(b);
 
-    a = (UChar*)calloc(asz*4 + 1, sizeof(UChar));
+    return (PyErr_Occurred()) ? NULL : Py_BuildValue("i", res);
-    b = (UChar*)calloc(bsz*4 + 1, sizeof(UChar));
-
-
-    if (a == NULL || b == NULL) return PyErr_NoMemory();
-
-    u_strFromUTF8(a, asz*4 + 1, NULL, a_, asz, &status);
-    u_strFromUTF8(b, bsz*4 + 1, NULL, b_, bsz, &status);
-    PyMem_Free(a_); PyMem_Free(b_);
-
-    if (U_SUCCESS(status))
-        res = ucol_strcoll(self->collator, a, -1, b, -1);
-
-    free(a); free(b);
-
-    return Py_BuildValue("i", res);
 } // }}}
 
 // Collator.find {{{
 static PyObject *
 icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) {
-    PyObject *a_, *b_;
+#if PY_VERSION_HEX >= 0x03030000 
-    int32_t asz, bsz;
+#error Not implemented for python >= 3.3
-    UChar *a, *b;
+#endif
-    wchar_t *aw, *bw;
+    PyObject *a_ = NULL, *b_ = NULL;
+    UChar *a = NULL, *b = NULL;
+    int32_t asz = 0, bsz = 0, pos = -1, length = -1;
     UErrorCode status = U_ZERO_ERROR;
     UStringSearch *search = NULL;
-    int32_t pos = -1, length = -1;
   
-    if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL;
+    if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL;
-    asz = (int32_t)PyUnicode_GetSize(a_); bsz = (int32_t)PyUnicode_GetSize(b_);
 
-    a = (UChar*)calloc(asz*4 + 2, sizeof(UChar));
+    a = python_to_icu(a_, &asz, 1);
-    b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar));
+    if (a == NULL) goto end;
-    aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t));
+    b = python_to_icu(b_, &bsz, 1);
-    bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t));
+    if (b == NULL) goto end;
 
-    if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory();
+    search = usearch_openFromCollator(a, asz, b, bsz, self->collator, NULL, &status);
-
-    PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1);
-    PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1);
-    u_strFromWCS(a, asz*4 + 1, NULL, aw, -1, &status);
-    u_strFromWCS(b, bsz*4 + 1, NULL, bw, -1, &status);
-
-    if (U_SUCCESS(status)) {
-        search = usearch_openFromCollator(a, -1, b, -1, self->collator, NULL, &status);
     if (U_SUCCESS(status)) {
         pos = usearch_first(search, &status);
-            if (pos != USEARCH_DONE) 
+        if (pos != USEARCH_DONE) {
             length = usearch_getMatchedLength(search);
-            else
+#ifdef Py_UNICODE_WIDE
-                pos = -1;
+            // We have to return number of unicode characters since the string
+            // could contain surrogate pairs which are represented as a single
+            // character in python wide builds
+            length = u_countChar32(b + pos, length);
+            pos = u_countChar32(b, pos);
+#endif
+        } else pos = -1;
     }
+end:
     if (search != NULL) usearch_close(search);
+    if (a != NULL) free(a);
+    if (b != NULL) free(b);
+
+    return (PyErr_Occurred()) ? NULL : Py_BuildValue("ii", pos, length);
+} // }}}
+
+// Collator.contains {{{
+static PyObject *
+icu_Collator_contains(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    PyObject *a_ = NULL, *b_ = NULL;
+    UChar *a = NULL, *b = NULL;
+    int32_t asz = 0, bsz = 0, pos = -1;
+    uint8_t found = 0;
+    UErrorCode status = U_ZERO_ERROR;
+    UStringSearch *search = NULL;
+  
+    if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL;
+
+    a = python_to_icu(a_, &asz, 1);
+    if (a == NULL) goto end;
+    if (asz == 0) { found = TRUE; goto end; }
+    b = python_to_icu(b_, &bsz, 1);
+    if (b == NULL) goto end;
+
+    search = usearch_openFromCollator(a, asz, b, bsz, self->collator, NULL, &status);
+    if (U_SUCCESS(status)) {
+        pos = usearch_first(search, &status);
+        if (pos != USEARCH_DONE) found = TRUE;
     }
+end:
+    if (search != NULL) usearch_close(search);
+    if (a != NULL) free(a);
+    if (b != NULL) free(b);
 
-    free(a); free(b); free(aw); free(bw);
+    if (PyErr_Occurred()) return NULL;
-
+    if (found) Py_RETURN_TRUE;
-    return Py_BuildValue("ii", pos, length);
+    Py_RETURN_FALSE;
 } // }}}
 
 // Collator.contractions {{{
 static PyObject *
 icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) {
     UErrorCode status = U_ZERO_ERROR;
-    UChar *str;
+    UChar *str = NULL;
     UChar32 start=0, end=0;
-    int32_t count = 0, len = 0, dlen = 0, i;
+    int32_t count = 0, len = 0, i;
     PyObject *ans = Py_None, *pbuf;
-    wchar_t *buf;
 
     if (self->contractions == NULL) {
         self->contractions = uset_open(1, 0);
@@ -269,107 +277,112 @@ icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs)
         self->contractions = ucol_getTailoredSet(self->collator, &status);
     }
     status = U_ZERO_ERROR; 
+    count = uset_getItemCount(self->contractions);
 
     str = (UChar*)calloc(100, sizeof(UChar));
-    buf = (wchar_t*)calloc(4*100+2, sizeof(wchar_t));
+    if (str == NULL) { PyErr_NoMemory(); goto end; }
-    if (str == NULL || buf == NULL) return PyErr_NoMemory();
-
-    count = uset_getItemCount(self->contractions);
     ans = PyTuple_New(count);
-    if (ans != NULL) {
+    if (ans == NULL) { goto end; }
+
     for (i = 0; i < count; i++) {
         len = uset_getItem(self->contractions, i, &start, &end, str, 1000, &status);
         if (len >= 2) {
             // We have a string
             status = U_ZERO_ERROR;
-                u_strToWCS(buf, 4*100 + 1, &dlen, str, len, &status);
+            pbuf = icu_to_python(str, len);
-                pbuf = PyUnicode_FromWideChar(buf, dlen);
+            if (pbuf == NULL) { Py_DECREF(ans); ans = NULL; goto end; }
-                if (pbuf == NULL) return PyErr_NoMemory();
             PyTuple_SetItem(ans, i, pbuf);
         } else {
             // Ranges dont make sense for contractions, ignore them
-                PyTuple_SetItem(ans, i, Py_None);
+            PyTuple_SetItem(ans, i, Py_None); Py_INCREF(Py_None);
         }
     }
-    }
+end:
-    free(str); free(buf);
+    if (str != NULL) free(str);
   
-    return Py_BuildValue("O", ans);
+    return ans;
 } // }}}
 
 // Collator.startswith {{{
 static PyObject *
 icu_Collator_startswith(icu_Collator *self, PyObject *args, PyObject *kwargs) {
-    PyObject *a_, *b_;
+    PyObject *a_ = NULL, *b_ = NULL;
-    int32_t asz, bsz;
+    int32_t asz = 0, bsz = 0;
-    int32_t actual_a, actual_b;
+    UChar *a = NULL, *b = NULL;
-    UChar *a, *b;
+    uint8_t ans = 0;
-    wchar_t *aw, *bw;
-    UErrorCode status = U_ZERO_ERROR;
-    int ans = 0;
   
-    if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL;
+    if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL;
-    asz = (int32_t)PyUnicode_GetSize(a_); bsz = (int32_t)PyUnicode_GetSize(b_);
-    if (asz < bsz) Py_RETURN_FALSE;
-    if (bsz == 0) Py_RETURN_TRUE;
 
-    a = (UChar*)calloc(asz*4 + 2, sizeof(UChar));
+    a = python_to_icu(a_, &asz, 1);
-    b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar));
+    if (a == NULL) goto end;
-    aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t));
+    b = python_to_icu(b_, &bsz, 1);
-    bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t));
+    if (b == NULL) goto end;
 
-    if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory();
+    if (asz < bsz) goto end;
+    if (bsz == 0) { ans = 1; goto end; }
     
-    actual_a = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1);
+    ans = ucol_equal(self->collator, a, bsz, b, bsz);
-    actual_b = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1);
-    if (actual_a > -1 && actual_b > -1) {
-        u_strFromWCS(a, asz*4 + 1, &actual_a, aw, -1, &status);
-        u_strFromWCS(b, bsz*4 + 1, &actual_b, bw, -1, &status);
 
-        if (U_SUCCESS(status) && ucol_equal(self->collator, a, actual_b, b, actual_b))
+end:
-            ans = 1;
+    if (a != NULL) free(a);
-    }
+    if (b != NULL) free(b);
 
-    free(a); free(b); free(aw); free(bw);
+    if (PyErr_Occurred()) return NULL;
-    if (ans) Py_RETURN_TRUE;
+    if (ans) { Py_RETURN_TRUE; }
     Py_RETURN_FALSE;
 } // }}}
 
-// Collator.startswith {{{
+// Collator.collation_order {{{
 static PyObject *
 icu_Collator_collation_order(icu_Collator *self, PyObject *args, PyObject *kwargs) {
-    PyObject *a_;
+    PyObject *a_ = NULL;
-    int32_t asz;
+    int32_t asz = 0;
-    int32_t actual_a;
+    UChar *a = NULL;
-    UChar *a;
-    wchar_t *aw;
     UErrorCode status = U_ZERO_ERROR;
     UCollationElements *iter = NULL;
     int order = 0, len = -1;
   
-    if (!PyArg_ParseTuple(args, "U", &a_)) return NULL;
+    if (!PyArg_ParseTuple(args, "O", &a_)) return NULL;
-    asz = (int32_t)PyUnicode_GetSize(a_); 
 
-    a = (UChar*)calloc(asz*4 + 2, sizeof(UChar));
+    a = python_to_icu(a_, &asz, 1);
-    aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t));
+    if (a == NULL) goto end;
 
-    if (a == NULL || aw == NULL ) return PyErr_NoMemory();
+    iter = ucol_openElements(self->collator, a, asz, &status);
-
+    if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); goto end; }
-    actual_a = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1);
-    if (actual_a > -1) {
-        u_strFromWCS(a, asz*4 + 1, &actual_a, aw, -1, &status);
-        iter = ucol_openElements(self->collator, a, actual_a, &status);
-        if (iter != NULL && U_SUCCESS(status)) {
     order = ucol_next(iter, &status);
     len = ucol_getOffset(iter);
-            ucol_closeElements(iter); iter = NULL;
+end:
-        }
+    if (iter != NULL) ucol_closeElements(iter); iter = NULL;
-    }
+    if (a != NULL) free(a);
-
+    if (PyErr_Occurred()) return NULL;
-    free(a); free(aw); 
     return Py_BuildValue("ii", order, len);
 } // }}}
 
+// Collator.upper_first {{{
+static PyObject *
+icu_Collator_get_upper_first(icu_Collator *self, void *closure) {
+    UErrorCode status = U_ZERO_ERROR;
+    UColAttributeValue val;
+
+    val = ucol_getAttribute(self->collator, UCOL_CASE_FIRST, &status);
+    if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); return NULL; }
+
+    if (val == UCOL_OFF) { Py_RETURN_NONE; }
+    if (val) {
+        Py_RETURN_TRUE;
+    }
+    Py_RETURN_FALSE;
+}
+
+static int
+icu_Collator_set_upper_first(icu_Collator *self, PyObject *val, void *closure) {
+    UErrorCode status = U_ZERO_ERROR;
+    ucol_setAttribute(self->collator, UCOL_CASE_FIRST, (val == Py_None) ? UCOL_OFF : ((PyObject_IsTrue(val)) ? UCOL_UPPER_FIRST : UCOL_LOWER_FIRST), &status);
+    if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); return -1; }
+    return 0;
+}
+// }}}
+
 static PyObject*
 icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs);
 
@@ -386,6 +399,10 @@ static PyMethodDef icu_Collator_methods[] = {
         "find(pattern, source) -> returns the position and length of the first occurrence of pattern in source. Returns (-1, -1) if not found."
     },
 
+    {"contains", (PyCFunction)icu_Collator_contains, METH_VARARGS,
+        "contains(pattern, source) -> return True iff the pattern was found in the source."
+    },
+
     {"contractions", (PyCFunction)icu_Collator_contractions, METH_VARARGS,
         "contractions() -> returns the contractions defined for this collator."
     },
@@ -411,6 +428,11 @@ static PyGetSetDef  icu_Collator_getsetters[] = {
      (char *)"Actual locale used by this collator.",
      NULL},
 
+    {(char *)"capsule", 
+     (getter)icu_Collator_capsule, NULL,
+     (char *)"A capsule enclosing the pointer to the ICU collator struct",
+     NULL},
+
     {(char *)"display_name", 
      (getter)icu_Collator_display_name, NULL,
      (char *)"Display name of this collator in English. The name reflects the actual data source used.",
@@ -421,6 +443,11 @@ static PyGetSetDef  icu_Collator_getsetters[] = {
      (char *)"The strength of this collator.",
      NULL},
 
+    {(char *)"upper_first",
+     (getter)icu_Collator_get_upper_first, (setter)icu_Collator_set_upper_first,
+     (char *)"Whether this collator should always put upper case letters before lower case. Values are: None - means use the tertiary strength of the letters. True - Always sort upper case before lower case. False - Always sort lower case before upper case.",
+     NULL},
+
     {(char *)"numeric",
      (getter)icu_Collator_get_numeric, (setter)icu_Collator_set_numeric,
      (char *)"If True the collator sorts contiguous digits as numbers rather than strings, so 2 will sort before 10.",
@@ -502,139 +529,45 @@ icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs)
 // }}}
 
 
-// upper {{{
+// change_case {{{
-static PyObject *
+
-icu_upper(PyObject *self, PyObject *args) {
+static PyObject* icu_change_case(PyObject *self, PyObject *args) {
-    char *input, *ans, *buf3 = NULL;
+    char *locale = NULL;
-    const char *loc;
+    PyObject *input = NULL, *result = NULL;
-    int32_t sz;
+    int which = UPPER_CASE;
-    UChar *buf, *buf2;
-    PyObject *ret;
     UErrorCode status = U_ZERO_ERROR;
+    UChar *input_buf = NULL, *output_buf = NULL;
+    int32_t sz = 0;
 
-
+    if (!PyArg_ParseTuple(args, "Oiz", &input, &which, &locale)) return NULL;
-    if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL;
+    if (locale == NULL) {
-    
+        PyErr_SetString(PyExc_NotImplementedError, "You must specify a locale");  // We deliberately use NotImplementedError so that this error can be unambiguously identified
-    sz = (int32_t)strlen(input);
+        return NULL;
-
-    buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
-    buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar));
-
-
-    if (buf == NULL || buf2 == NULL) return PyErr_NoMemory();
-
-    u_strFromUTF8(buf, sz*4, NULL, input, sz, &status);
-    u_strToUpper(buf2, sz*8, buf, -1, loc, &status);
-
-    ans = input;
-    sz = u_strlen(buf2);
-    free(buf);
-
-    if (U_SUCCESS(status) && sz > 0) {
-        buf3 = (char*)calloc(sz*5+1, sizeof(char));
-        if (buf3 == NULL) return PyErr_NoMemory();
-        u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status);
-        if (U_SUCCESS(status)) ans = buf3;
     }
 
-    ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace");
+    input_buf = python_to_icu(input, &sz, 1);
-    if (ret == NULL) return PyErr_NoMemory();
+    if (input_buf == NULL) goto end;
+    output_buf = (UChar*) calloc(3 * sz, sizeof(UChar));
+    if (output_buf == NULL) { PyErr_NoMemory(); goto end; }
 
-    free(buf2);
+    switch (which) {
-    if (buf3 != NULL) free(buf3);
+        case TITLE_CASE:
-    PyMem_Free(input);
+            sz = u_strToTitle(output_buf, 3 * sz, input_buf, sz, NULL, locale, &status);
-
+            break;
-    return ret;
+        case UPPER_CASE:
-} // }}}
+            sz = u_strToUpper(output_buf, 3 * sz, input_buf, sz, locale, &status);
-
+            break;
-// lower {{{
+        default:
-static PyObject *
+            sz = u_strToLower(output_buf, 3 * sz, input_buf, sz, locale, &status);
-icu_lower(PyObject *self, PyObject *args) {
-    char *input, *ans, *buf3 = NULL;
-    const char *loc;
-    int32_t sz;
-    UChar *buf, *buf2;
-    PyObject *ret;
-    UErrorCode status = U_ZERO_ERROR;
-  
-
-    if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL;
-    
-    sz = (int32_t)strlen(input);
-
-    buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
-    buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar));
-
-
-    if (buf == NULL || buf2 == NULL) return PyErr_NoMemory();
-
-    u_strFromUTF8(buf, sz*4, NULL, input, sz, &status);
-    u_strToLower(buf2, sz*8, buf, -1, loc, &status);
-
-    ans = input;
-    sz = u_strlen(buf2);
-    free(buf);
-
-    if (U_SUCCESS(status) && sz > 0) {
-        buf3 = (char*)calloc(sz*5+1, sizeof(char));
-        if (buf3 == NULL) return PyErr_NoMemory();
-        u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status);
-        if (U_SUCCESS(status)) ans = buf3;
     }
+    if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); goto end; }
+    result = icu_to_python(output_buf, sz);
 
-    ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace");
+end:
-    if (ret == NULL) return PyErr_NoMemory();
+    if (input_buf != NULL) free(input_buf);
+    if (output_buf != NULL) free(output_buf);
+    return result;
 
-    free(buf2);
-    if (buf3 != NULL) free(buf3);
-    PyMem_Free(input);
-
-    return ret;
-} // }}}
-
-// title {{{
-static PyObject *
-icu_title(PyObject *self, PyObject *args) {
-    char *input, *ans, *buf3 = NULL;
-    const char *loc;
-    int32_t sz;
-    UChar *buf, *buf2;
-    PyObject *ret;
-    UErrorCode status = U_ZERO_ERROR;
-  
-
-    if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL;
-    
-    sz = (int32_t)strlen(input);
-
-    buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar));
-    buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar));
-
-
-    if (buf == NULL || buf2 == NULL) return PyErr_NoMemory();
-
-    u_strFromUTF8(buf, sz*4, NULL, input, sz, &status);
-    u_strToTitle(buf2, sz*8, buf, -1, NULL, loc, &status);
-
-    ans = input;
-    sz = u_strlen(buf2);
-    free(buf);
-
-    if (U_SUCCESS(status) && sz > 0) {
-        buf3 = (char*)calloc(sz*5+1, sizeof(char));
-        if (buf3 == NULL) return PyErr_NoMemory();
-        u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status);
-        if (U_SUCCESS(status)) ans = buf3;
-    }
-
-    ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace");
-    if (ret == NULL) return PyErr_NoMemory();
-
-    free(buf2);
-    if (buf3 != NULL) free(buf3);
-    PyMem_Free(input);
-
-    return ret;
 } // }}}
 
 // set_default_encoding {{{
@@ -651,7 +584,7 @@ icu_set_default_encoding(PyObject *self, PyObject *args) {
 }
 // }}}
 
-// set_default_encoding {{{
+// set_filesystem_encoding {{{
 static PyObject *
 icu_set_filesystem_encoding(PyObject *self, PyObject *args) {
     char *encoding;
@@ -663,7 +596,7 @@ icu_set_filesystem_encoding(PyObject *self, PyObject *args) {
 }
 // }}}
 
-// set_default_encoding {{{
+// get_available_transliterators {{{
 static PyObject *
 icu_get_available_transliterators(PyObject *self, PyObject *args) {
     PyObject *ans, *l;
@@ -824,16 +757,8 @@ icu_roundtrip(PyObject *self, PyObject *args) {
 
 // Module initialization {{{
 static PyMethodDef icu_methods[] = {
-    {"upper", icu_upper, METH_VARARGS,
+    {"change_case", icu_change_case, METH_VARARGS,
-        "upper(locale, unicode object) -> upper cased unicode object using locale rules."
+        "change_case(unicode object, which, locale) -> change case to one of UPPER_CASE, LOWER_CASE, TITLE_CASE"
-    },
-
-    {"lower", icu_lower, METH_VARARGS,
-        "lower(locale, unicode object) -> lower cased unicode object using locale rules."
-    },
-
-    {"title", icu_title, METH_VARARGS,
-        "title(locale, unicode object) -> Title cased unicode object using locale rules."
     },
 
     {"set_default_encoding", icu_set_default_encoding, METH_VARARGS,
@@ -935,5 +860,9 @@ initicu(void)
     ADDUCONST(UNORM_NFKC);
     ADDUCONST(UNORM_FCD);
 
+    ADDUCONST(UPPER_CASE);
+    ADDUCONST(LOWER_CASE);
+    ADDUCONST(TITLE_CASE);
+
 }
 // }}}

src/calibre/utils/icu.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -7,535 +9,251 @@ __docformat__ = 'restructuredtext en'
 
 # Setup code {{{
 import sys
-from functools import partial
 
 from calibre.constants import plugins
 from calibre.utils.config_base import tweaks
 
-_icu = _collator = _primary_collator = _sort_collator = _numeric_collator = None
+_locale = _collator = _primary_collator = _sort_collator = _numeric_collator = _case_sensitive_collator = None
-_locale = None
 
 _none = u''
 _none2 = b''
+_cmap = {}
 
-def get_locale():
+_icu, err = plugins['icu']
-    global _locale
+if _icu is None:
+    raise RuntimeError('Failed to load icu with error: %s' % err)
+del err
+icu_unicode_version = getattr(_icu, 'unicode_version', None)
+_nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NFKD', 'NONE', 'DEFAULT', 'FCD')}
+
+try:
+    senc = sys.getdefaultencoding()
+    if not senc or senc.lower() == b'ascii':
+        _icu.set_default_encoding(b'utf-8')
+    del senc
+except:
+    import traceback
+    traceback.print_exc()
+
+try:
+    fenc = sys.getfilesystemencoding()
+    if not fenc or fenc.lower() == b'ascii':
+        _icu.set_filesystem_encoding(b'utf-8')
+    del fenc
+except:
+    import traceback
+    traceback.print_exc()
+
+def collator():
+    global _collator, _locale
+    if _collator is None:
         if _locale is None:
             from calibre.utils.localization import get_lang
             if tweaks['locale_for_sorting']:
                 _locale = tweaks['locale_for_sorting']
             else:
                 _locale = get_lang()
-    return _locale
+        try:
-
+            _collator = _icu.Collator(_locale)
-def load_icu():
+        except Exception as e:
-    global _icu
+            print ('Failed to load collator for locale: %r with error %r, using English' % (_locale, e))
-    if _icu is None:
+            _collator = _icu.Collator('en')
-        _icu = plugins['icu'][0]
-        if _icu is None:
-            print 'Loading ICU failed with: ', plugins['icu'][1]
-        else:
-            if not getattr(_icu, 'ok', False):
-                print 'icu not ok'
-                _icu = None
-    return _icu
-
-def load_collator():
-    'The default collator for most locales takes both case and accented letters into account'
-    global _collator
-    if _collator is None:
-        icu = load_icu()
-        if icu is not None:
-            _collator = icu.Collator(get_locale())
     return _collator
 
+def change_locale(locale=None):
+    global _locale, _collator, _primary_collator, _sort_collator, _numeric_collator, _case_sensitive_collator
+    _collator = _primary_collator = _sort_collator = _numeric_collator = _case_sensitive_collator = None
+    _locale = locale
+
 def primary_collator():
     'Ignores case differences and accented characters'
     global _primary_collator
     if _primary_collator is None:
-        _primary_collator = _collator.clone()
+        _primary_collator = collator().clone()
         _primary_collator.strength = _icu.UCOL_PRIMARY
     return _primary_collator
 
 def sort_collator():
-    'Ignores case differences and recognizes numbers in strings'
+    'Ignores case differences and recognizes numbers in strings (if the tweak is set)'
     global _sort_collator
     if _sort_collator is None:
-        _sort_collator = _collator.clone()
+        _sort_collator = collator().clone()
         _sort_collator.strength = _icu.UCOL_SECONDARY
-        if tweaks['numeric_collation']:
+        _sort_collator.numeric = tweaks['numeric_collation']
-            try:
-                _sort_collator.numeric = True
-            except AttributeError:
-                pass
     return _sort_collator
 
-def py_sort_key(obj):
-    if not obj:
-        return _none
-    return obj.lower()
-
-def icu_sort_key(collator, obj):
-    if not obj:
-        return _none2
-    try:
-        try:
-            return _sort_collator.sort_key(obj)
-        except AttributeError:
-            return sort_collator().sort_key(obj)
-    except TypeError:
-        if isinstance(obj, unicode):
-            obj = obj.replace(u'\0', u'')
-        else:
-            obj = obj.replace(b'\0', b'')
-        return _sort_collator.sort_key(obj)
-
 def numeric_collator():
+    'Uses natural sorting for numbers inside strings so something2 will sort before something10'
     global _numeric_collator
-    _numeric_collator = _collator.clone()
+    if _numeric_collator is None:
+        _numeric_collator = collator().clone()
         _numeric_collator.strength = _icu.UCOL_SECONDARY
         _numeric_collator.numeric = True
     return _numeric_collator
 
-def numeric_sort_key(obj):
+def case_sensitive_collator():
-    'Uses natural sorting for numbers inside strings so something2 will sort before something10'
+    'Always sorts upper case letter before lower case'
-    if not obj:
+    global _case_sensitive_collator
-        return _none2
+    if _case_sensitive_collator is None:
+        _case_sensitive_collator = collator().clone()
+        _case_sensitive_collator.numeric = sort_collator().numeric
+        _case_sensitive_collator.upper_first = True
+    return _case_sensitive_collator
+
+# Templates that will be used to generate various concrete
+# function implementations based on different collators, to allow lazy loading
+# of collators, with maximum runtime performance
+
+_sort_key_template = '''
+def {name}(obj):
     try:
         try:
-            return _numeric_collator.sort_key(obj)
+            return {collator}.{func}(obj)
         except AttributeError:
-            return numeric_collator().sort_key(obj)
+            return {collator_func}().{func}(obj)
     except TypeError:
-        if isinstance(obj, unicode):
+        if isinstance(obj, bytes):
-            obj = obj.replace(u'\0', u'')
-        else:
-            obj = obj.replace(b'\0', b'')
-        return _numeric_collator.sort_key(obj)
-
-def icu_change_case(upper, locale, obj):
-    func = _icu.upper if upper else _icu.lower
             try:
-        return func(locale, obj)
+                obj = obj.decode(sys.getdefaultencoding())
-    except TypeError:
+            except ValueError:
-        if isinstance(obj, unicode):
+                return obj
-            obj = obj.replace(u'\0', u'')
+            return {collator}.{func}(obj)
-        else:
+    return b''
-            obj = obj.replace(b'\0', b'')
+'''
-        return func(locale, obj)
 
-def py_find(pattern, source):
+_strcmp_template = '''
-    pos = source.find(pattern)
+def {name}(a, b):
-    if pos > -1:
+    try:
-        return pos, len(pattern)
+        try:
-    return -1, -1
+            return {collator}.{func}(a, b)
+        except AttributeError:
+            return {collator_func}().{func}(a, b)
+    except TypeError:
+        if isinstance(a, bytes):
+            try:
+                a = a.decode(sys.getdefaultencoding())
+            except ValueError:
+                return cmp(a, b)
+        elif a is None:
+            a = u''
+        if isinstance(b, bytes):
+            try:
+                b = b.decode(sys.getdefaultencoding())
+            except ValueError:
+                return cmp(a, b)
+        elif b is None:
+            b = u''
+        return {collator}.{func}(a, b)
+'''
+
+_change_case_template = '''
+def {name}(x):
+    try:
+        try:
+            return _icu.change_case(x, _icu.{which}, _locale)
+        except NotImplementedError:
+            collator()  # sets _locale
+            return _icu.change_case(x, _icu.{which}, _locale)
+    except TypeError:
+        if isinstance(x, bytes):
+            try:
+                x = x.decode(sys.getdefaultencoding())
+            except ValueError:
+                return x
+            return _icu.change_case(x, _icu.{which}, _locale)
+        raise
+'''
+
+def _make_func(template, name, **kwargs):
+    l = globals()
+    kwargs['name'] = name
+    kwargs['func'] = kwargs.get('func', 'sort_key')
+    exec template.format(**kwargs) in l
+    return l[name]
+
+
+# }}}
+
+################# The string functions ########################################
+sort_key = _make_func(_sort_key_template, 'sort_key', collator='_sort_collator', collator_func='sort_collator')
+
+numeric_sort_key = _make_func(_sort_key_template, 'numeric_sort_key', collator='_numeric_collator', collator_func='numeric_collator')
+
+primary_sort_key = _make_func(_sort_key_template, 'primary_sort_key', collator='_primary_collator', collator_func='primary_collator')
+
+case_sensitive_sort_key = _make_func(_sort_key_template, 'case_sensitive_sort_key',
+                                     collator='_case_sensitive_collator', collator_func='case_sensitive_collator')
+
+collation_order = _make_func(_sort_key_template, 'collation_order', collator='_sort_collator', collator_func='sort_collator', func='collation_order')
+
+strcmp = _make_func(_strcmp_template, 'strcmp', collator='_sort_collator', collator_func='sort_collator', func='strcmp')
+
+case_sensitive_strcmp = _make_func(
+    _strcmp_template, 'case_sensitive_strcmp', collator='_case_sensitive_collator', collator_func='case_sensitive_collator', func='strcmp')
+
+primary_strcmp = _make_func(_strcmp_template, 'primary_strcmp', collator='_primary_collator', collator_func='primary_collator', func='strcmp')
+
+upper = _make_func(_change_case_template, 'upper', which='UPPER_CASE')
+
+lower = _make_func(_change_case_template, 'lower', which='LOWER_CASE')
+
+title_case = _make_func(_change_case_template, 'title_case', which='TITLE_CASE')
+
+def capitalize(x):
+    try:
+        return upper(x[0]) + lower(x[1:])
+    except (IndexError, TypeError, AttributeError):
+        return x
+
+find = _make_func(_strcmp_template, 'find', collator='_collator', collator_func='collator', func='find')
+
+primary_find = _make_func(_strcmp_template, 'primary_find', collator='_primary_collator', collator_func='primary_collator', func='find')
+
+contains = _make_func(_strcmp_template, 'contains', collator='_collator', collator_func='collator', func='contains')
+
+primary_contains = _make_func(_strcmp_template, 'primary_contains', collator='_primary_collator', collator_func='primary_collator', func='contains')
+
+startswith = _make_func(_strcmp_template, 'startswith', collator='_collator', collator_func='collator', func='startswith')
+
+primary_startswith = _make_func(_strcmp_template, 'primary_startswith', collator='_primary_collator', collator_func='primary_collator', func='startswith')
+
+safe_chr = _icu.chr
 
 def character_name(string):
-    try:
     try:
         return _icu.character_name(unicode(string)) or None
-        except AttributeError:
-            import unicodedata
-            return unicodedata.name(unicode(string)[0], None)
     except (TypeError, ValueError, KeyError):
         pass
 
 def character_name_from_code(code):
-    try:
     try:
         return _icu.character_name_from_code(code) or ''
-        except AttributeError:
-            import unicodedata
-            return unicodedata.name(py_safe_chr(code), '')
     except (TypeError, ValueError, KeyError):
         return ''
 
-if sys.maxunicode >= 0x10ffff:
-    try:
-        py_safe_chr = unichr
-    except NameError:
-        py_safe_chr = chr
-else:
-    def py_safe_chr(i):
-        # Narrow builds of python cannot represent code point > 0xffff as a
-        # single character, so we need our own implementation of unichr
-        # that returns them as a surrogate pair
-        return (b"\U%s" % (hex(i)[2:].zfill(8))).decode('unicode-escape')
-
-def safe_chr(code):
-    try:
-        return _icu.chr(code)
-    except AttributeError:
-        return py_safe_chr(code)
-
 def normalize(text, mode='NFC'):
     # This is very slightly slower than using unicodedata.normalize, so stick with
     # that unless you have very good reasons not too. Also, it's speed
     # decreases on wide python builds, where conversion to/from ICU's string
     # representation is slower.
-    try:
     return _icu.normalize(_nmodes[mode], unicode(text))
-    except (AttributeError, KeyError):
-        import unicodedata
-        return unicodedata.normalize(mode, unicode(text))
 
-def icu_find(collator, pattern, source):
+def contractions(col=None):
-    try:
-        return collator.find(pattern, source)
-    except TypeError:
-        return collator.find(unicode(pattern), unicode(source))
-
-def icu_startswith(collator, a, b):
-    try:
-        return collator.startswith(a, b)
-    except TypeError:
-        return collator.startswith(unicode(a), unicode(b))
-
-def py_case_sensitive_sort_key(obj):
-    if not obj:
-        return _none
-    return obj
-
-def icu_case_sensitive_sort_key(collator, obj):
-    if not obj:
-        return _none2
-    return collator.sort_key(obj)
-
-def icu_strcmp(collator, a, b):
-    return collator.strcmp(lower(a), lower(b))
-
-def py_strcmp(a, b):
-    return cmp(a.lower(), b.lower())
-
-def icu_case_sensitive_strcmp(collator, a, b):
-    return collator.strcmp(a, b)
-
-def icu_capitalize(s):
-    s = lower(s)
-    return s.replace(s[0], upper(s[0]), 1) if s else s
-
-_cmap = {}
-def icu_contractions(collator):
     global _cmap
+    col = col or _collator
+    if col is None:
+        col = collator()
     ans = _cmap.get(collator, None)
     if ans is None:
-        ans = collator.contractions()
+        ans = col.contractions()
-        ans = frozenset(filter(None, ans)) if ans else {}
+        ans = frozenset(filter(None, ans))
-        _cmap[collator] = ans
+        _cmap[col] = ans
     return ans
 
-def icu_collation_order(collator, a):
-    try:
-        return collator.collation_order(a)
-    except TypeError:
-        return collator.collation_order(unicode(a))
-
-load_icu()
-load_collator()
-_icu_not_ok = _icu is None or _collator is None
-icu_unicode_version = getattr(_icu, 'unicode_version', None)
-_nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NFKD', 'NONE', 'DEFAULT', 'FCD')}
-
-try:
-    senc = sys.getdefaultencoding()
-    if not senc or senc.lower() == 'ascii':
-        _icu.set_default_encoding('utf-8')
-    del senc
-except:
-    pass
-
-try:
-    fenc = sys.getfilesystemencoding()
-    if not fenc or fenc.lower() == 'ascii':
-        _icu.set_filesystem_encoding('utf-8')
-    del fenc
-except:
-    pass
-
-
-# }}}
-
-################# The string functions ########################################
-
-sort_key = py_sort_key if _icu_not_ok else partial(icu_sort_key, _collator)
-
-strcmp = py_strcmp if _icu_not_ok else partial(icu_strcmp, _collator)
-
-case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \
-        partial(icu_case_sensitive_sort_key, _collator)
-
-case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp
-
-upper = (lambda s: s.upper()) if _icu_not_ok else \
-    partial(icu_change_case, True, get_locale())
-
-lower = (lambda s: s.lower()) if _icu_not_ok else \
-    partial(icu_change_case, False, get_locale())
-
-title_case = (lambda s: s.title()) if _icu_not_ok else \
-    partial(_icu.title, get_locale())
-
-capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \
-    (lambda s: icu_capitalize(s))
-
-find = (py_find if _icu_not_ok else partial(icu_find, _collator))
-
-contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions,
-    _collator)))
-
-def primary_strcmp(a, b):
-    'strcmp that ignores case and accents on letters'
-    if _icu_not_ok:
-        from calibre.utils.filenames import ascii_text
-        return py_strcmp(ascii_text(a), ascii_text(b))
-    try:
-        return _primary_collator.strcmp(a, b)
-    except AttributeError:
-        return primary_collator().strcmp(a, b)
-
-def primary_find(pat, src):
-    'find that ignores case and accents on letters'
-    if _icu_not_ok:
-        from calibre.utils.filenames import ascii_text
-        return py_find(ascii_text(pat), ascii_text(src))
-    return primary_icu_find(pat, src)
-
-def primary_icu_find(pat, src):
-    try:
-        return icu_find(_primary_collator, pat, src)
-    except AttributeError:
-        return icu_find(primary_collator(), pat, src)
-
-def primary_sort_key(val):
-    'A sort key that ignores case and diacritics'
-    if _icu_not_ok:
-        from calibre.utils.filenames import ascii_text
-        return ascii_text(val).lower()
-    try:
-        return _primary_collator.sort_key(val)
-    except AttributeError:
-        return primary_collator().sort_key(val)
-
-def primary_startswith(a, b):
-    if _icu_not_ok:
-        from calibre.utils.filenames import ascii_text
-        return ascii_text(a).lower().startswith(ascii_text(b).lower())
-    try:
-        return icu_startswith(_primary_collator, a, b)
-    except AttributeError:
-        return icu_startswith(primary_collator(), a, b)
-
-def collation_order(a):
-    if _icu_not_ok:
-        return (ord(a[0]), 1) if a else (0, 0)
-    try:
-        return icu_collation_order(_sort_collator, a)
-    except AttributeError:
-        return icu_collation_order(sort_collator(), a)
 
 ################################################################################
 
-def test():  # {{{
-    from calibre import prints
-    # Data {{{
-    german = '''
-    Sonntag
-Montag
-Dienstag
-Januar
-Februar
-März
-Fuße
-Fluße
-Flusse
-flusse
-fluße
-flüße
-flüsse
-'''
-    german_good = '''
-    Dienstag
-Februar
-flusse
-Flusse
-fluße
-Fluße
-flüsse
-flüße
-Fuße
-Januar
-März
-Montag
-Sonntag'''
-    french = '''
-dimanche
-lundi
-mardi
-janvier
-février
-mars
-déjà
-Meme
-deja
-même
-dejà
-bpef
-bœg
-Boef
-Mémé
-bœf
-boef
-bnef
-pêche
-pèché
-pêché
-pêche
-pêché'''
-    french_good = '''
-            bnef
-        boef
-        Boef
-        bœf
-        bœg
-        bpef
-        deja
-        dejà
-        déjà
-        dimanche
-        février
-        janvier
-        lundi
-        mardi
-        mars
-        Meme
-        Mémé
-        même
-        pèché
-        pêche
-        pêche
-        pêché
-        pêché'''
-    # }}}
-
-    def create(l):
-        l = l.decode('utf-8').splitlines()
-        return [x.strip() for x in l if x.strip()]
-
-    def test_strcmp(entries):
-        for x in entries:
-            for y in entries:
-                if strcmp(x, y) != cmp(sort_key(x), sort_key(y)):
-                    print 'strcmp failed for %r, %r'%(x, y)
-
-    german = create(german)
-    c = _icu.Collator('de')
-    c.numeric = True
-    gs = list(sorted(german, key=c.sort_key))
-    if gs != create(german_good):
-        print 'German sorting failed'
-        return
-    print
-    french = create(french)
-    c = _icu.Collator('fr')
-    c.numeric = True
-    fs = list(sorted(french, key=c.sort_key))
-    if fs != create(french_good):
-        print 'French sorting failed (note that French fails with icu < 4.6)'
-        return
-    test_strcmp(german + french)
-
-    print '\nTesting case transforms in current locale'
-    from calibre.utils.titlecase import titlecase
-    for x in ('a', 'Alice\'s code', 'macdonald\'s machine', '02 the wars'):
-        print 'Upper:     ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8')
-        print 'Lower:     ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8')
-        print 'Title:     ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8'), 'titlecase:', titlecase(x).encode('utf-8')
-        print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8')
-        print
-
-    print '\nTesting primary collation'
-    for k, v in {u'pèché': u'peche', u'flüße':u'Flusse',
-            u'Štepánek':u'ŠtepaneK'}.iteritems():
-        if primary_strcmp(k, v) != 0:
-            prints('primary_strcmp() failed with %s != %s'%(k, v))
-            return
-        if primary_find(v, u' '+k)[0] != 1:
-            prints('primary_find() failed with %s not in %s'%(v, k))
-            return
-
-    n = character_name(safe_chr(0x1f431))
-    if n != u'CAT FACE':
-        raise ValueError('Failed to get correct character name for 0x1f431: %r != %r' % n, u'CAT FACE')
-
-    global _primary_collator
-    orig = _primary_collator
-    _primary_collator = _icu.Collator('es')
-    if primary_strcmp(u'peña', u'pena') == 0:
-        print 'Primary collation in Spanish locale failed'
-        return
-    _primary_collator = orig
-
-    print '\nTesting contractions'
-    c = _icu.Collator('cs')
-    if icu_contractions(c) != frozenset([u'Z\u030c', u'z\u030c', u'Ch',
-        u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH',
-        u'S\u030c', u'R\u030c']):
-        print 'Contractions for the Czech language failed'
-        return
-
-    print '\nTesting startswith'
-    p = primary_startswith
-    if (not p('asd', 'asd') or not p('asd', 'A') or
-            not p('x', '')):
-        print 'startswith() failed'
-        return
-
-    print '\nTesting collation_order()'
-    for group in [
-        ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'),
-        ('calibre', 'Charon', 'Collins'),
-        ('01', '1'),
-        ('1', '11', '13'),
-    ]:
-        last = None
-        for x in group:
-            val = icu_collation_order(sort_collator(), x)
-            if val[1] != 1:
-                prints('collation_order() returned incorrect length for', x)
-            if last is None:
-                last = val
-            else:
-                if val != last:
-                    prints('collation_order() returned incorrect value for', x)
-            last = val
-
-# }}}
-
-def test_roundtrip():
-    for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'):
-        rp = _icu.roundtrip(r)
-        if rp != r:
-            raise ValueError(u'Roundtripping failed: %r != %r' % (r, rp))
-
-def test_normalize_performance():
-    import os
-    if not os.path.exists('t.txt'):
-        return
-    raw = open('t.txt', 'rb').read().decode('utf-8')
-    print (len(raw))
-    import time, unicodedata
-    st = time.time()
-    count = 100
-    for i in xrange(count):
-        normalize(raw)
-    print ('ICU time:', time.time() - st)
-    st = time.time()
-    for i in xrange(count):
-        unicodedata.normalize('NFC', unicode(raw))
-    print ('py time:', time.time() - st)
-
 if __name__ == '__main__':
-    test_roundtrip()
+    from calibre.utils.icu_test import run
-    test_normalize_performance()
+    run(verbosity=4)
-    test()
 

src/calibre/utils/icu_calibre_utils.h

@@ -21,7 +21,10 @@
 #include <unicode/utrans.h>
 #include <unicode/unorm.h>
 
-#if PY_VERSION_HEX < 0x03030000 
+#if PY_VERSION_HEX >= 0x03030000 
+#error Not implemented for python >= 3.3
+#endif
+
 // Roundtripping will need to be implemented differently for python 3.3+ where strings are stored with variable widths
 
 #ifndef NO_PYTHON_TO_ICU
@@ -67,5 +70,4 @@ static PyObject* icu_to_python(UChar *src, int32_t sz) {
 }
 #endif
 
-#endif
 

src/calibre/utils/icu_test.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import unittest, sys
+from contextlib import contextmanager
+
+import calibre.utils.icu as icu
+
+
+@contextmanager
+def make_collation_func(name, locale, numeric=True, template='_sort_key_template', func='strcmp'):
+    c = icu._icu.Collator(locale)
+    cname = '%s_test_collator%s' % (name, template)
+    setattr(icu, cname, c)
+    c.numeric = numeric
+    yield icu._make_func(getattr(icu, template), name, collator=cname, collator_func='not_used_xxx', func=func)
+    delattr(icu, cname)
+
+class TestICU(unittest.TestCase):
+
+    ae = unittest.TestCase.assertEqual
+
+    def setUp(self):
+        icu.change_locale('en')
+
+    def test_sorting(self):
+        ' Test the various sorting APIs '
+        german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split()
+        german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split()
+        french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split()
+        french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split()  # noqa
+
+        # Test corner cases
+        sort_key = icu.sort_key
+        s = '\U0001f431'
+        self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key')
+        self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself')
+        self.ae(b'', sort_key(None))
+        self.ae(0, icu.strcmp(None, b''))
+        self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding())))
+
+        # Test locales
+        with make_collation_func('dsk', 'de', func='sort_key') as dsk:
+            self.ae(german_good, sorted(german, key=dsk))
+            with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp:
+                for x in german:
+                    for y in german:
+                        self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y))
+
+        with make_collation_func('fsk', 'fr', func='sort_key') as fsk:
+            self.ae(french_good, sorted(french, key=fsk))
+            with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp:
+                for x in french:
+                    for y in french:
+                        self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y))
+
+        with make_collation_func('ssk', 'es', func='sort_key') as ssk:
+            self.assertNotEqual(ssk('peña'), ssk('pena'))
+            with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp:
+                self.assertNotEqual(0, scmp('pena', 'peña'))
+
+        for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems():
+            self.ae(0, icu.primary_strcmp(k, v))
+
+        # Test different types of collation
+        self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa'))
+        self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11'))
+        self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a'))
+        self.ae(0, icu.strcmp('a', 'A'))
+        self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A'))
+        self.ae(0, icu.primary_strcmp('ä', 'A'))
+
+    def test_change_case(self):
+        ' Test the various ways of changing the case '
+        from calibre.utils.titlecase import titlecase
+        # Test corner cases
+        self.ae('A', icu.upper(b'a'))
+        for x in ('', None, False, 1):
+            self.ae(x, icu.capitalize(x))
+
+        for x in ('a', 'Alice\'s code', 'macdonald\'s machIne', '02 the wars'):
+            self.ae(icu.upper(x), x.upper())
+            self.ae(icu.lower(x), x.lower())
+            # ICU's title case algorithm is different from ours, when there are
+            # capitals inside words
+            self.ae(icu.title_case(x), titlecase(x).replace('machIne', 'Machine'))
+            self.ae(icu.capitalize(x), x[0].upper() + x[1:].lower())
+
+    def test_find(self):
+        ' Test searching for substrings '
+        self.ae((1, 1), icu.find(b'a', b'1ab'))
+        self.ae((1, 1 if sys.maxunicode >= 0x10ffff else 2), icu.find('\U0001f431', 'x\U0001f431x'))
+        self.ae((1 if sys.maxunicode >= 0x10ffff else 2, 1), icu.find('y', '\U0001f431y'))
+        self.ae((0, 4), icu.primary_find('pena', 'peña'))
+        for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems():
+            self.ae((1, len(k)), icu.primary_find(v, ' ' + k), 'Failed to find %s in %s' % (v, k))
+        self.assertTrue(icu.startswith(b'abc', b'ab'))
+        self.assertTrue(icu.startswith('abc', 'abc'))
+        self.assertFalse(icu.startswith('xyz', 'a'))
+        self.assertTrue(icu.startswith('xxx', ''))
+        self.assertTrue(icu.primary_startswith('pena', 'peña'))
+        self.assertTrue(icu.contains('\U0001f431', '\U0001f431'))
+        self.assertTrue(icu.contains('something', 'some other something else'))
+        self.assertTrue(icu.contains('', 'a'))
+        self.assertTrue(icu.contains('', ''))
+        self.assertFalse(icu.contains('xxx', 'xx'))
+        self.assertTrue(icu.primary_contains('pena', 'peña'))
+
+    def test_collation_order(self):
+        'Testing collation ordering'
+        for group in [
+            ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'),
+            ('01', '1'),
+            ('1', '11', '13'),
+        ]:
+            last = None
+            for x in group:
+                order, length = icu.numeric_collator().collation_order(x)
+                if last is not None:
+                    self.ae(last, order)
+                last = order
+
+    def test_roundtrip(self):
+        for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'):
+            self.ae(r, icu._icu.roundtrip(r))
+
+    def test_character_name(self):
+        self.ae(icu.character_name('\U0001f431'), 'CAT FACE')
+
+    def test_contractions(self):
+        c = icu._icu.Collator('cs')
+        self.ae(icu.contractions(c), frozenset({u'Z\u030c', u'z\u030c', u'Ch',
+            u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH',
+            u'S\u030c', u'R\u030c'}))
+
+class TestRunner(unittest.main):
+
+    def createTests(self):
+        tl = unittest.TestLoader()
+        self.test = tl.loadTestsFromTestCase(TestICU)
+
+def run(verbosity=4):
+    TestRunner(verbosity=verbosity, exit=False)
+
+def test_build():
+    result = TestRunner(verbosity=0, buffer=True, catchbreak=True, failfast=True, argv=sys.argv[:1], exit=False).result
+    if not result.wasSuccessful():
+        raise SystemExit(1)
+
+if __name__ == '__main__':
+    run(verbosity=4)
+

src/calibre/utils/ipc/__init__.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 import os, errno
 from threading import Thread
 
-from calibre.constants import iswindows, get_windows_username
+from calibre.constants import iswindows, get_windows_username, islinux
 
 ADDRESS = None
 
@@ -37,11 +37,14 @@ def gui_socket_address():
                 if user:
                     ADDRESS += '-' + user[:100] + 'x'
         else:
-            from tempfile import gettempdir
-            tmp = gettempdir()
             user = os.environ.get('USER', '')
             if not user:
                 user = os.path.basename(os.path.expanduser('~'))
+            if islinux:
+                ADDRESS = (u'\0%s-calibre-gui.socket' % user).encode('ascii')
+            else:
+                from tempfile import gettempdir
+                tmp = gettempdir()
                 ADDRESS = os.path.join(tmp, user+'-calibre-gui.socket')
     return ADDRESS
 

src/calibre/utils/ipc/server.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, cPickle, time, tempfile
+import sys, os, cPickle, time, tempfile, errno
 from math import ceil
 from threading import Thread, RLock
 from Queue import Queue, Empty
@@ -18,7 +18,7 @@ from calibre.utils.ipc import eintr_retry_call
 from calibre.utils.ipc.launch import Worker
 from calibre.utils.ipc.worker import PARALLEL_FUNCS
 from calibre import detect_ncpus as cpu_count
-from calibre.constants import iswindows, DEBUG
+from calibre.constants import iswindows, DEBUG, islinux
 from calibre.ptempfile import base_dir
 
 _counter = 0
@@ -84,6 +84,35 @@ class ConnectedWorker(Thread):
 class CriticalError(Exception):
     pass
 
+_name_counter = 0
+
+if islinux:
+    def create_listener(authkey, backlog=4):
+        # Use abstract named sockets on linux to avoid creating unnecessary temp files
+        global _name_counter
+        prefix = u'\0calibre-ipc-listener-%d-%%d' % os.getpid()
+        while True:
+            _name_counter += 1
+            address = (prefix % _name_counter).encode('ascii')
+            try:
+                l = Listener(address=address, authkey=authkey, backlog=backlog)
+                if hasattr(l._listener._unlink, 'cancel'):
+                    # multiprocessing tries to call unlink even on abstract
+                    # named sockets, prevent it from doing so.
+                    l._listener._unlink.cancel()
+                return address, l
+            except EnvironmentError as err:
+                if err.errno == errno.EADDRINUSE:
+                    continue
+                raise
+else:
+    def create_listener(authkey, backlog=4):
+        address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        if iswindows and address[1] == ':':
+            address = address[2:]
+        listener = Listener(address=address, authkey=authkey, backlog=backlog)
+        return address, listener
+
 class Server(Thread):
 
     def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
@@ -99,11 +128,7 @@ class Server(Thread):
         self.pool_size = limit if pool_size is None else pool_size
         self.notify_on_job_done = notify_on_job_done
         self.auth_key = os.urandom(32)
-        self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        self.address, self.listener = create_listener(self.auth_key, backlog=4)
-        if iswindows and self.address[1] == ':':
-            self.address = self.address[2:]
-        self.listener = Listener(address=self.address,
-                authkey=self.auth_key, backlog=4)
         self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
         self.kill_queue = Queue()
         self.waiting_jobs = []
@@ -162,7 +187,6 @@ class Server(Thread):
         w = self.launch_worker(gui=gui, redirect_output=redirect_output)
         w.start_job(job)
 
-
     def run(self):
         while True:
             try:
@@ -280,8 +304,6 @@ class Server(Thread):
             pos += delta
         return ans
 
-
-
     def close(self):
         try:
             self.add_jobs_queue.put(None)

src/calibre/utils/ipc/simple_worker.py

@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, cPickle, traceback, time, importlib
 from binascii import hexlify, unhexlify
-from multiprocessing.connection import Listener, arbitrary_address, Client
+from multiprocessing.connection import Client
 from threading import Thread
 from contextlib import closing
 
@@ -117,11 +117,9 @@ def communicate(ans, worker, listener, args, timeout=300, heartbeat=None,
     ans['result'] = cw.res['result']
 
 def create_worker(env, priority='normal', cwd=None, func='main'):
-    address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+    from calibre.utils.ipc.server import create_listener
-    if iswindows and address[1] == ':':
-        address = address[2:]
     auth_key = os.urandom(32)
-    listener = Listener(address=address, authkey=auth_key)
+    address, listener = create_listener(auth_key)
 
     env = dict(env)
     env.update({

src/calibre/utils/lock.py

@@ -8,7 +8,7 @@ Secure access to locked files from multiple processes.
 
 from calibre.constants import iswindows, __appname__, \
                               win32api, win32event, winerror, fcntl
-import time, atexit, os
+import time, atexit, os, stat
 
 class LockError(Exception):
     pass
@@ -105,6 +105,12 @@ class WindowsExclFile(object):
     def closed(self):
         return self._handle is None
 
+def unix_open(path):
+    # We cannot use open(a+b) directly because Fedora apparently ships with a
+    # broken libc that causes seek(0) followed by truncate() to not work for
+    # files with O_APPEND set.
+    fd = os.open(path, os.O_RDWR | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
+    return os.fdopen(fd, 'r+b')
 
 class ExclusiveFile(object):
 
@@ -113,7 +119,7 @@ class ExclusiveFile(object):
         self.timeout = timeout
 
     def __enter__(self):
-        self.file = WindowsExclFile(self.path, self.timeout) if iswindows else open(self.path, 'a+b')
+        self.file = WindowsExclFile(self.path, self.timeout) if iswindows else unix_open(self.path)
         self.file.seek(0)
         timeout = self.timeout
         if not iswindows:

src/calibre/utils/matcher.c

@@ -155,28 +155,34 @@ static double calc_score_for_char(MatchInfo *m, UChar32 last, UChar32 current, i
 }
 
 static void convert_positions(int32_t *positions, int32_t *final_positions, UChar *string, int32_t char_len, int32_t byte_len, double score) {
+#if PY_VERSION_HEX >= 0x03030000 
+#error Not implemented for python >= 3.3
+#endif
+
     // The positions array stores character positions as byte offsets in string, convert them into character offsets
     int32_t i, *end;
 
-    if (score == 0.0) {
+    if (score == 0.0) { for (i = 0; i < char_len; i++) final_positions[i] = -1; return; }
-        for (i = 0; i < char_len; i++) final_positions[i] = -1;
-        return;
-    }
 
     end = final_positions + char_len;
     for (i = 0; i < byte_len && final_positions < end; i++) {
         if (positions[i] == -1) continue;
+#ifdef Py_UNICODE_WIDE
         *final_positions = u_countChar32(string, positions[i]);
+#else
+        *final_positions = positions[i];
+#endif
         final_positions += 1;
     }
 }
 
-static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions) {
+static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions, UStringSearch **searches) {
-    UChar32 nc, hc, lc;
+    UChar32 hc, lc;
-    UChar *p;
     double final_score = 0.0, score = 0.0, score_for_char = 0.0;
     int32_t pos, i, j, hidx, nidx, last_idx, distance, *positions = final_positions + m->needle_len;
     MemoryItem mem = {0};
+    UStringSearch *search = NULL;
+    UErrorCode status = U_ZERO_ERROR;
 
     stack_push(stack, 0, 0, 0, 0.0, final_positions);
 
@@ -187,11 +193,14 @@ static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions)
             // No memoized result, calculate the score
             for (i = nidx; i < m->needle_len;) {
                 nidx = i;
-                U16_NEXT(m->needle, i, m->needle_len, nc); // i now points to next char in needle 
+                U16_FWD_1(m->needle, i, m->needle_len);// i now points to next char in needle 
-                if (m->haystack_len - hidx < m->needle_len - nidx) { score = 0.0; break; }
+                search = searches[nidx];
-                p = u_strchr32(m->haystack + hidx, nc);  // TODO: Use primary collation for the find
+                if (search == NULL || m->haystack_len - hidx < m->needle_len - nidx) { score = 0.0; break; }
-                if (p == NULL) { score = 0.0; break; }
+                status = U_ZERO_ERROR; // We ignore any errors as we already know that hidx is correct
-                pos = (int32_t)(p - m->haystack);
+                usearch_setOffset(search, hidx, &status);
+                status = U_ZERO_ERROR;
+                pos = usearch_next(search, &status);
+                if (pos == USEARCH_DONE) { score = 0.0; break; } // No matches found
                 distance = u_countChar32(m->haystack + last_idx, pos - last_idx);  
                 if (distance <= 1) score_for_char = m->max_score_per_char;
                 else {
@@ -222,8 +231,30 @@ static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions)
     return final_score;
 }
 
+static bool create_searches(UStringSearch **searches, UChar *haystack, int32_t haystack_len, UChar *needle, int32_t needle_len, UCollator *collator) {
+    int32_t i = 0, pos = 0;
+    UErrorCode status = U_ZERO_ERROR;
 
-static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UChar *needle, Match *match_results, int32_t *final_positions, int32_t needle_char_len, UChar *level1, UChar *level2, UChar *level3) {
+    while (i < needle_len) {
+        pos = i;
+        U16_FWD_1(needle, i, needle_len);
+        if (pos == i) break;
+        searches[pos] = usearch_openFromCollator(needle + pos, i - pos, haystack, haystack_len, collator, NULL, &status);
+        if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); searches[pos] = NULL; return FALSE; }
+    }
+
+    return TRUE;
+}
+
+static void free_searches(UStringSearch **searches, int32_t count) {
+    int32_t i = 0;
+    for (i = 0; i < count; i++) {
+        if (searches[i] != NULL) usearch_close(searches[i]);
+        searches[i] = NULL;
+    }
+}
+
+static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UChar *needle, Match *match_results, int32_t *final_positions, int32_t needle_char_len, UCollator *collator, UChar *level1, UChar *level2, UChar *level3) {
     Stack stack = {0};
     int32_t i = 0, maxhl = 0; 
     int32_t r = 0, *positions = NULL;
@@ -231,6 +262,7 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh
     bool ok = FALSE;
     MemoryItem ***memo = NULL;
     int32_t needle_len = u_strlen(needle);
+    UStringSearch **searches = NULL;
 
     if (needle_len <= 0 || item_count <= 0) {
         for (i = 0; i < (int32_t)item_count; i++) match_results[i].score = 0.0;
@@ -240,7 +272,8 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh
 
     matches = (MatchInfo*)calloc(item_count, sizeof(MatchInfo));
     positions = (int32_t*)calloc(2*needle_len, sizeof(int32_t)); // One set of positions is the final answer and one set is working space
-    if (matches == NULL || positions == NULL) {PyErr_NoMemory(); goto end;}
+    searches = (UStringSearch**) calloc(needle_len, sizeof(UStringSearch*));
+    if (matches == NULL || positions == NULL || searches == NULL) {PyErr_NoMemory(); goto end;}
 
     for (i = 0; i < (int32_t)item_count; i++) {
         matches[i].haystack = items[i];
@@ -265,14 +298,14 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh
     if (stack.items == NULL || memo == NULL) {PyErr_NoMemory(); goto end;}
 
     for (i = 0; i < (int32_t)item_count; i++) {
-        for (r = 0; r < needle_len; r++) {
+        for (r = 0; r < needle_len; r++)  positions[r] = -1;
-            positions[r] = -1;
-        }
         stack_clear(&stack);
         clear_memory(memo, needle_len, matches[i].haystack_len);
+        free_searches(searches, needle_len);
+        if (!create_searches(searches, matches[i].haystack, matches[i].haystack_len, needle, needle_len, collator)) goto end;
         matches[i].memo = memo;
-        match_results[i].score = process_item(&matches[i], &stack, positions);
+        match_results[i].score = process_item(&matches[i], &stack, positions, searches);
-        convert_positions(positions, final_positions + i, matches[i].haystack, needle_char_len, needle_len, match_results[i].score);
+        convert_positions(positions, final_positions + i * needle_char_len, matches[i].haystack, needle_char_len, needle_len, match_results[i].score);
     }
 
     ok = TRUE;
@@ -281,6 +314,7 @@ end:
     nullfree(stack.items);
     nullfree(matches);
     nullfree(memo);
+    if (searches != NULL) { free_searches(searches, needle_len); nullfree(searches); }
     return ok;
 }
 
@@ -296,6 +330,7 @@ typedef struct {
     UChar *level1;
     UChar *level2;
     UChar *level3;
+    UCollator *collator;
 
 } Matcher;
 
@@ -308,6 +343,7 @@ static void free_matcher(Matcher *self) {
     }
     nullfree(self->items); nullfree(self->item_lengths); 
     nullfree(self->level1); nullfree(self->level2); nullfree(self->level3);
+    if (self->collator != NULL) ucol_close(self->collator); self->collator = NULL;
 }
 static void
 Matcher_dealloc(Matcher* self)
@@ -320,10 +356,21 @@ Matcher_dealloc(Matcher* self)
 static int
 Matcher_init(Matcher *self, PyObject *args, PyObject *kwds)
 {
-    PyObject *items = NULL, *p = NULL, *py_items = NULL, *level1 = NULL, *level2 = NULL, *level3 = NULL;
+    PyObject *items = NULL, *p = NULL, *py_items = NULL, *level1 = NULL, *level2 = NULL, *level3 = NULL, *collator = NULL;
     int32_t i = 0;
+    UErrorCode status = U_ZERO_ERROR;
+    UCollator *col = NULL;
+
+    if (!PyArg_ParseTuple(args, "OOOOO", &items, &collator, &level1, &level2, &level3)) return -1;
+    
+    // Clone the passed in collator (cloning is needed as collators are not thread safe)
+    if (!PyCapsule_CheckExact(collator)) { PyErr_SetString(PyExc_TypeError, "Collator must be a capsule"); return -1; }
+    col = (UCollator*)PyCapsule_GetPointer(collator, NULL);
+    if (col == NULL) return -1;
+    self->collator = ucol_safeClone(col, NULL, NULL, &status);
+    col = NULL;
+    if (U_FAILURE(status)) { self->collator = NULL; PyErr_SetString(PyExc_ValueError, u_errorName(status)); return -1; }
 
-    if (!PyArg_ParseTuple(args, "OOOO", &items, &level1, &level2, &level3)) return -1;
     py_items = PySequence_Fast(items,  "Must pass in two sequence objects");
     if (py_items == NULL) goto end;
     self->item_count = (uint32_t)PySequence_Size(items);
@@ -378,7 +425,7 @@ Matcher_calculate_scores(Matcher *self, PyObject *args) {
     }
 
     Py_BEGIN_ALLOW_THREADS;
-    ok = match(self->items, self->item_lengths, self->item_count, needle, matches, final_positions, needle_char_len, self->level1, self->level2, self->level3);
+    ok = match(self->items, self->item_lengths, self->item_count, needle, matches, final_positions, needle_char_len, self->collator, self->level1, self->level2, self->level3);
     Py_END_ALLOW_THREADS;
 
     if (ok) {
@@ -386,7 +433,7 @@ Matcher_calculate_scores(Matcher *self, PyObject *args) {
             score = PyFloat_FromDouble(matches[i].score);
             if (score == NULL) { PyErr_NoMemory(); goto end; }
             PyTuple_SET_ITEM(items, (Py_ssize_t)i, score);
-            p = final_positions + i;
+            p = final_positions + (i * needle_char_len);
             for (j = 0; j < needle_char_len; j++) {
                 score = PyInt_FromLong((long)p[j]);
                 if (score == NULL) { PyErr_NoMemory(); goto end; }

src/calibre/utils/matcher.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import atexit, os, sys
+from math import ceil
+from unicodedata import normalize
+from threading import Thread, Lock
+from Queue import Queue
+from operator import itemgetter
+from collections import OrderedDict
+from itertools import islice
+
+from itertools import izip
+from future_builtins import map
+
+from calibre import detect_ncpus as cpu_count, as_unicode
+from calibre.constants import plugins, filesystem_encoding
+from calibre.utils.icu import primary_sort_key, primary_find, primary_collator
+
+DEFAULT_LEVEL1 = '/'
+DEFAULT_LEVEL2 = '-_ 0123456789'
+DEFAULT_LEVEL3 = '.'
+
+class PluginFailed(RuntimeError):
+    pass
+
+class Worker(Thread):
+
+    daemon = True
+
+    def __init__(self, requests, results):
+        Thread.__init__(self)
+        self.requests, self.results = requests, results
+        atexit.register(lambda : requests.put(None))
+
+    def run(self):
+        while True:
+            x = self.requests.get()
+            if x is None:
+                break
+            try:
+                i, scorer, query = x
+                self.results.put((True, (i, scorer(query))))
+            except Exception as e:
+                self.results.put((False, as_unicode(e)))
+                # import traceback
+                # traceback.print_exc()
+wlock = Lock()
+workers = []
+
+def split(tasks, pool_size):
+    '''
+    Split a list into a list of sub lists, with the number of sub lists being
+    no more than pool_size. Each sublist contains
+    2-tuples of the form (i, x) where x is an element from the original list
+    and i is the index of the element x in the original list.
+    '''
+    ans, count = [], 0
+    delta = int(ceil(len(tasks)/pool_size))
+    while tasks:
+        section = [(count+i, task) for i, task in enumerate(tasks[:delta])]
+        tasks = tasks[delta:]
+        count += len(section)
+        ans.append(section)
+    return ans
+
+def default_scorer(*args, **kwargs):
+    try:
+        return CScorer(*args, **kwargs)
+    except PluginFailed:
+        return PyScorer(*args, **kwargs)
+
+class Matcher(object):
+
+    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3, scorer=None):
+        with wlock:
+            if not workers:
+                requests, results = Queue(), Queue()
+                w = [Worker(requests, results) for i in range(max(1, cpu_count()))]
+                [x.start() for x in w]
+                workers.extend(w)
+        items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items))
+        self.items = items = tuple(items)
+        tasks = split(items, len(workers))
+        self.task_maps = [{j:i for j, (i, _) in enumerate(task)} for task in tasks]
+        scorer = scorer or default_scorer
+        self.scorers = [scorer(tuple(map(itemgetter(1), task_items))) for task_items in tasks]
+        self.sort_keys = None
+
+    def __call__(self, query, limit=None):
+        query = normalize('NFC', unicode(query))
+        with wlock:
+            for i, scorer in enumerate(self.scorers):
+                workers[0].requests.put((i, scorer, query))
+            if self.sort_keys is None:
+                self.sort_keys = {i:primary_sort_key(x) for i, x in enumerate(self.items)}
+            num = len(self.task_maps)
+            scores, positions = {}, {}
+            error = None
+            while num > 0:
+                ok, x = workers[0].results.get()
+                num -= 1
+                if ok:
+                    task_num, vals = x
+                    task_map = self.task_maps[task_num]
+                    for i, (score, pos) in enumerate(vals):
+                        item = task_map[i]
+                        scores[item] = score
+                        positions[item] = pos
+                else:
+                    error = x
+
+        if error is not None:
+            raise Exception('Failed to score items: %s' % error)
+        items = sorted(((-scores[i], item, positions[i]) for i, item in enumerate(self.items)),
+                       key=itemgetter(0))
+        if limit is not None:
+            del items[limit:]
+        return OrderedDict(x[1:] for x in filter(itemgetter(0), items))
+
+def get_items_from_dir(basedir, acceptq=lambda x: True):
+    if isinstance(basedir, bytes):
+        basedir = basedir.decode(filesystem_encoding)
+    relsep = os.sep != '/'
+    for dirpath, dirnames, filenames in os.walk(basedir):
+        for f in filenames:
+            x = os.path.join(dirpath, f)
+            if acceptq(x):
+                x = os.path.relpath(x, basedir)
+                if relsep:
+                    x = x.replace(os.sep, '/')
+                yield x
+
+class FilesystemMatcher(Matcher):
+
+    def __init__(self, basedir, *args, **kwargs):
+        Matcher.__init__(self, get_items_from_dir(basedir), *args, **kwargs)
+
+# Python implementation of the scoring algorithm {{{
+def calc_score_for_char(ctx, prev, current, distance):
+    factor = 1.0
+    ans = ctx.max_score_per_char
+
+    if prev in ctx.level1:
+        factor = 0.9
+    elif prev in ctx.level2 or (icu_lower(prev) == prev and icu_upper(current) == current):
+        factor = 0.8
+    elif prev in ctx.level3:
+        factor = 0.7
+    else:
+        factor = (1.0 / distance) * 0.75
+
+    return ans * factor
+
+def process_item(ctx, haystack, needle):
+    # non-recursive implementation using a stack
+    stack = [(0, 0, 0, 0, [-1]*len(needle))]
+    final_score, final_positions = stack[0][-2:]
+    push, pop = stack.append, stack.pop
+    while stack:
+        hidx, nidx, last_idx, score, positions = pop()
+        key = (hidx, nidx, last_idx)
+        mem = ctx.memory.get(key, None)
+        if mem is None:
+            for i in xrange(nidx, len(needle)):
+                n = needle[i]
+                if (len(haystack) - hidx < len(needle) - i):
+                    score = 0
+                    break
+                pos = primary_find(n, haystack[hidx:])[0]
+                if pos == -1:
+                    score = 0
+                    break
+                pos += hidx
+
+                distance = pos - last_idx
+                score_for_char = ctx.max_score_per_char if distance <= 1 else calc_score_for_char(ctx, haystack[pos-1], haystack[pos], distance)
+                hidx = pos + 1
+                push((hidx, i, last_idx, score, list(positions)))
+                last_idx = positions[i] = pos
+                score += score_for_char
+            ctx.memory[key] = (score, positions)
+        else:
+            score, positions = mem
+        if score > final_score:
+            final_score = score
+            final_positions = positions
+    return final_score, final_positions
+
+class PyScorer(object):
+    __slots__ = ('level1', 'level2', 'level3', 'max_score_per_char', 'items', 'memory')
+
+    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
+        self.level1, self.level2, self.level3 = level1, level2, level3
+        self.max_score_per_char = 0
+        self.items = items
+
+    def __call__(self, needle):
+        for item in self.items:
+            self.max_score_per_char = (1.0 / len(item) + 1.0 / len(needle)) / 2.0
+            self.memory = {}
+            yield process_item(self, item, needle)
+# }}}
+
+class CScorer(object):
+
+    def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3):
+        speedup, err = plugins['matcher']
+        if speedup is None:
+            raise PluginFailed('Failed to load the matcher plugin with error: %s' % err)
+        self.m = speedup.Matcher(items, primary_collator().capsule, unicode(level1), unicode(level2), unicode(level3))
+
+    def __call__(self, query):
+        scores, positions = self.m.calculate_scores(query)
+        for score, pos in izip(scores, positions):
+            yield score, pos
+
+def test():
+    import unittest
+
+    class Test(unittest.TestCase):
+
+        def test_mem_leaks(self):
+            import gc
+            from calibre.utils.mem import get_memory as memory
+            m = Matcher(['a'], scorer=CScorer)
+            m('a')
+            def doit(c):
+                m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',], scorer=CScorer)
+                m('one')
+            start = memory()
+            for i in xrange(10):
+                doit(str(i))
+            gc.collect()
+            used10 = memory() - start
+            start = memory()
+            for i in xrange(100):
+                doit(str(i))
+            gc.collect()
+            used100 = memory() - start
+            self.assertLessEqual(used100, 2 * used10)
+
+        def test_non_bmp(self):
+            raw = '_\U0001f431-'
+            m = Matcher([raw], scorer=CScorer)
+            positions = next(m(raw).itervalues())
+            self.assertEqual(positions, (0, 1, (2 if sys.maxunicode >= 0x10ffff else 3)))
+
+    class TestRunner(unittest.main):
+
+        def createTests(self):
+            tl = unittest.TestLoader()
+            self.test = tl.loadTestsFromTestCase(Test)
+
+    TestRunner(verbosity=4)
+
+if sys.maxunicode >= 0x10ffff:
+    get_char = lambda string, pos: string[pos]
+else:
+    def get_char(string, pos):
+        chs = 2 if ('\ud800' <= string[pos] <= '\udbff') else 1  # UTF-16 surrogate pair in python narrow builds
+        return string[pos:pos+chs]
+
+def main(basedir=None, query=None):
+    from calibre import prints
+    from calibre.utils.terminal import ColoredStream
+    if basedir is None:
+        try:
+            basedir = raw_input('Enter directory to scan [%s]: ' % os.getcwdu()).decode(sys.stdin.encoding).strip() or os.getcwdu()
+        except (EOFError, KeyboardInterrupt):
+            return
+    m = FilesystemMatcher(basedir)
+    emph = ColoredStream(sys.stdout, fg='red', bold=True)
+    while True:
+        if query is None:
+            try:
+                query = raw_input('Enter query: ').decode(sys.stdin.encoding)
+            except (EOFError, KeyboardInterrupt):
+                break
+            if not query:
+                break
+        for path, positions in islice(m(query).iteritems(), 0, 10):
+            positions = list(positions)
+            p = 0
+            while positions:
+                pos = positions.pop(0)
+                if pos == -1:
+                    continue
+                prints(path[p:pos], end='')
+                ch = get_char(path, pos)
+                with emph:
+                    prints(ch, end='')
+                p = pos + len(ch)
+            prints(path[p:])
+        query = None
+
+if __name__ == '__main__':
+    # main(basedir='/t', query='ns')
+    # test()
+    main()