0.9.30+

recipes/dot_net.recipe

@@ -1,32 +1,37 @@
-# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from calibre.web.feeds.news import BasicNewsRecipe
 import re
 
-class NetMagazineRecipe (BasicNewsRecipe):
-   __author__ = u'Marc Busqué <marc@lamarciana.com>'
-   __url__ = 'http://www.lamarciana.com'
-   __version__ = '1.0'
-   __license__   = 'GPL v3'
-   __copyright__ = u'2012, Marc Busqué <marc@lamarciana.com>'
-   title = u'.net magazine'
-   description = u'net is the world’s best-selling magazine for web designers and developers, featuring tutorials from leading agencies, interviews with the web’s biggest names, and agenda-setting features on the hottest issues affecting the internet today.'
-   language = 'en'
-   tags = 'web development, software'
-   oldest_article = 7
-   remove_empty_feeds = True
-   no_stylesheets = True
-   cover_url = u'http://media.netmagazine.futurecdn.net/sites/all/themes/netmag/logo.png'
-   keep_only_tags = [
-         dict(name='article', attrs={'class': re.compile('^node.*$', re.IGNORECASE)})
-         ]
-   remove_tags = [
-         dict(name='span', attrs={'class': 'comment-count'}),
-         dict(name='div', attrs={'class': 'item-list share-links'}),
-         dict(name='footer'),
-         ]
-   remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height', 'style']
-   extra_css = 'img {max-width: 100%; display: block; margin: auto;} .captioned-image div {text-align: center; font-style: italic;}'
+class dotnetMagazine (BasicNewsRecipe):
+    __author__ = u'Bonni Salles'
+    __version__ = '1.0'
+    __license__   = 'GPL v3'
+    __copyright__ = u'2013, Bonni Salles'
+    title                 = '.net magazine'
+    oldest_article        = 7
+    no_stylesheets        = True
+    encoding              = 'utf8'
+    use_embedded_content  = False
+    language              = 'en'
+    remove_empty_feeds    = True
+    extra_css             = ' body{font-family: Arial,Helvetica,sans-serif } img{margin-bottom: 0.4em} '
+    cover_url = u'http://media.netmagazine.futurecdn.net/sites/all/themes/netmag/logo.png'
+
+    remove_tags_after = dict(name='footer', id=lambda x:not x)
+    remove_tags_before = dict(name='header', id=lambda x:not x)
+
+    remove_tags = [
+         dict(name='div', attrs={'class': 'item-list'}),
+         dict(name='h4', attrs={'class': 'std-hdr'}),
+         dict(name='div', attrs={'class': 'item-list share-links'}), #removes share links
+         dict(name=['script', 'noscript']),
+         dict(name='div', attrs={'id': 'comments-form'}), #comment these out if you want the comments to show
+         dict(name='div', attrs={'id': re.compile('advertorial_block_($|| )')}),
+         dict(name='div', attrs={'id': 'right-col'}),
+         dict(name='div', attrs={'id': 'comments'}), #comment these out if you want the comments to show
+         dict(name='div', attrs={'class': 'item-list related-content'}),
 
-   feeds = [
-         (u'.net', u'http://feeds.feedburner.com/net/topstories'),
          ]
+
+    feeds = [
+               (u'net', u'http://feeds.feedburner.com/net/topstories')
+            ]

recipes/nrc_next.recipe

@@ -0,0 +1,75 @@
+#!/usr/bin/env  python2
+# -*- coding: utf-8 -*-
+# Based on veezh's original recipe, Kovid Goyal's New York Times recipe and Snaabs nrc Handelsblad recipe
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Niels Giesen'
+
+'''
+www.nrc.nl
+'''
+import os, zipfile
+import time
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ptempfile import PersistentTemporaryFile
+
+
+class NRCNext(BasicNewsRecipe):
+
+    title = u'nrc•next'
+    description = u'De ePaper-versie van nrc•next'
+    language = 'nl'
+    lang = 'nl-NL'
+    needs_subscription = True
+
+    __author__ = 'Niels Giesen'
+
+    conversion_options = {
+        'no_default_epub_cover' : True
+    }
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser(self)
+        if self.username is not None and self.password is not None:
+            br.open('http://login.nrc.nl/login')
+            br.select_form(nr=0)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    def build_index(self):
+
+        today = time.strftime("%Y%m%d")
+
+        domain = "http://digitaleeditie.nrc.nl"
+
+        url = domain + "/digitaleeditie/helekrant/epub/nn_" + today + ".epub"
+        #print url
+
+        try:
+            br = self.get_browser()
+            f = br.open(url)
+        except:
+            self.report_progress(0,_('Kan niet inloggen om editie te downloaden'))
+            raise ValueError('Krant van vandaag nog niet beschikbaar')
+
+        tmp = PersistentTemporaryFile(suffix='.epub')
+        self.report_progress(0,_('downloading epub'))
+        tmp.write(f.read())
+        f.close()
+        br.close()
+        if zipfile.is_zipfile(tmp):
+            try:
+                zfile = zipfile.ZipFile(tmp.name, 'r')
+                zfile.extractall(self.output_dir)
+                self.report_progress(0,_('extracting epub'))
+            except zipfile.BadZipfile:
+                self.report_progress(0,_('BadZip error, continuing'))
+
+        tmp.close()
+        index = os.path.join(self.output_dir, 'metadata.opf')
+
+        self.report_progress(1,_('epub downloaded and extracted'))
+
+        return index

resources/default_tweaks.py

@@ -32,7 +32,7 @@ defaults.
 # Set the use_series_auto_increment_tweak_when_importing tweak to True to
 # use the above values when importing/adding books. If this tweak is set to
 # False (the default) then the series number will be set to 1 if it is not
-# explicitly set to during the import. If set to True, then the
+# explicitly set during the import. If set to True, then the
 # series index will be set according to the series_index_auto_increment setting.
 # Note that the use_series_auto_increment_tweak_when_importing tweak is used
 # only when a value is not provided during import. If the importing regular

src/calibre/customize/builtins.py

@@ -1548,12 +1548,13 @@ class StoreNextoStore(StoreBase):
 
 class StoreNookUKStore(StoreBase):
     name = 'Nook UK'
-    author = 'John Schember'
-    description = u'Barnes & Noble S.à r.l, a subsidiary of Barnes & Noble, Inc., a leading retailer of content, digital media and educational products, is proud to bring the award-winning NOOK® reading experience and a leading digital bookstore to the UK.'  # noqa
+    author = 'Charles Haley'
+    description = u'Barnes & Noble S.A.R.L, a subsidiary of Barnes & Noble, Inc., a leading retailer of content, digital media and educational products, is proud to bring the award-winning NOOK reading experience and a leading digital bookstore to the UK.'  # noqa
     actual_plugin = 'calibre.gui2.store.stores.nook_uk_plugin:NookUKStore'
 
     headquarters = 'UK'
     formats = ['NOOK']
+    affiliate = True
 
 class StoreOpenBooksStore(StoreBase):
     name = 'Open Books'

src/calibre/devices/interface.py

@@ -107,6 +107,12 @@ class DevicePlugin(Plugin):
     #: :meth:`set_user_blacklisted_devices`
     ASK_TO_ALLOW_CONNECT = False
 
+    #: Set this to a dictionary of the form {'title':title, 'msg':msg, 'det_msg':detailed_msg} to have calibre popup
+    #: a message to the user after some callbacks are run (currently only upload_books).
+    #: Be careful to not spam the user with too many messages. This variable is checked after *every* callback,
+    #: so only set it when you really need to.
+    user_feedback_after_callback = None
+
     @classmethod
     def get_gui_name(cls):
         if hasattr(cls, 'gui_name'):
@@ -157,16 +163,15 @@ class DevicePlugin(Plugin):
                 if (vid in device_id or vidd in device_id) and \
                    (pid in device_id or pidd in device_id) and \
                    self.test_bcd_windows(device_id, bcd):
-                       if debug:
-                           self.print_usb_device_info(device_id)
-                       if only_presence or self.can_handle_windows(device_id, debug=debug):
-                           try:
-                               bcd = int(device_id.rpartition(
-                                   'rev_')[-1].replace(':', 'a'), 16)
-                           except:
-                               bcd = None
-                           return True, (vendor_id, product_id, bcd, None,
-                                   None, None)
+                        if debug:
+                            self.print_usb_device_info(device_id)
+                        if only_presence or self.can_handle_windows(device_id, debug=debug):
+                            try:
+                                bcd = int(device_id.rpartition(
+                                            'rev_')[-1].replace(':', 'a'), 16)
+                            except:
+                                bcd = None
+                            return True, (vendor_id, product_id, bcd, None, None, None)
         return False, None
 
     def test_bcd(self, bcdDevice, bcd):
@@ -638,7 +643,6 @@ class DevicePlugin(Plugin):
         '''
         device_prefs.set_overrides()
 
-
     # Dynamic control interface.
     # The following methods are probably called on the GUI thread. Any driver
     # that implements these methods must take pains to be thread safe, because

src/calibre/ebooks/conversion/plumber.py

@@ -77,7 +77,7 @@ class Plumber(object):
 
     def __init__(self, input, output, log, report_progress=DummyReporter(),
             dummy=False, merge_plugin_recs=True, abort_after_input_dump=False,
-            override_input_metadata=False):
+            override_input_metadata=False, for_regex_wizard=False):
         '''
         :param input: Path to input file.
         :param output: Path to output file/directory
@@ -87,6 +87,7 @@ class Plumber(object):
         if isbytestring(output):
             output = output.decode(filesystem_encoding)
         self.original_input_arg = input
+        self.for_regex_wizard = for_regex_wizard
         self.input = os.path.abspath(input)
         self.output = os.path.abspath(output)
         self.log = log
@@ -123,7 +124,7 @@ OptionRecommendation(name='input_profile',
                    'conversion system information on how to interpret '
                    'various information in the input document. For '
                    'example resolution dependent lengths (i.e. lengths in '
-                   'pixels). Choices are:')+\
+                   'pixels). Choices are:')+
                         ', '.join([x.short_name for x in input_profiles()])
         ),
 
@@ -135,7 +136,7 @@ OptionRecommendation(name='output_profile',
                    'created document for the specified device. In some cases, '
                    'an output profile is required to produce documents that '
                    'will work on a device. For example EPUB on the SONY reader. '
-                   'Choices are:') + \
+                   'Choices are:') +
                            ', '.join([x.short_name for x in output_profiles()])
         ),
 
@@ -490,7 +491,7 @@ OptionRecommendation(name='asciiize',
             'cases where there are multiple representations of a character '
             '(characters shared by Chinese and Japanese for instance) the '
             'representation based on the current calibre interface language will be '
-            'used.')%\
+            'used.')%
             u'\u041c\u0438\u0445\u0430\u0438\u043b '
             u'\u0413\u043e\u0440\u0431\u0430\u0447\u0451\u0432'
 )
@@ -711,7 +712,6 @@ OptionRecommendation(name='search_replace',
         self.input_fmt = input_fmt
         self.output_fmt = output_fmt
 
-
         self.all_format_options = set()
         self.input_options = set()
         self.output_options = set()
@@ -775,7 +775,7 @@ OptionRecommendation(name='search_replace',
         if not html_files:
             raise ValueError(_('Could not find an ebook inside the archive'))
         html_files = [(f, os.stat(f).st_size) for f in html_files]
-        html_files.sort(cmp = lambda x, y: cmp(x[1], y[1]))
+        html_files.sort(cmp=lambda x, y: cmp(x[1], y[1]))
         html_files = [f[0] for f in html_files]
         for q in ('toc', 'index'):
             for f in html_files:
@@ -783,8 +783,6 @@ OptionRecommendation(name='search_replace',
                     return f, os.path.splitext(f)[1].lower()[1:]
         return html_files[-1], os.path.splitext(html_files[-1])[1].lower()[1:]
 
-
-
     def get_option_by_name(self, name):
         for group in (self.input_options, self.pipeline_options,
                       self.output_options, self.all_format_options):
@@ -956,7 +954,6 @@ OptionRecommendation(name='search_replace',
 
         self.log.info('Input debug saved to:', out_dir)
 
-
     def run(self):
         '''
         Run the conversion pipeline
@@ -965,10 +962,12 @@ OptionRecommendation(name='search_replace',
         self.setup_options()
         if self.opts.verbose:
             self.log.filter_level = self.log.DEBUG
+        if self.for_regex_wizard and hasattr(self.opts, 'no_process'):
+            self.opts.no_process = True
         self.flush()
         import cssutils, logging
         cssutils.log.setLevel(logging.WARN)
-        get_types_map() # Ensure the mimetypes module is intialized
+        get_types_map()  # Ensure the mimetypes module is intialized
 
         if self.opts.debug_pipeline is not None:
             self.opts.verbose = max(self.opts.verbose, 4)
@@ -1003,6 +1002,8 @@ OptionRecommendation(name='search_replace',
         self.ui_reporter(0.01, _('Converting input to HTML...'))
         ir = CompositeProgressReporter(0.01, 0.34, self.ui_reporter)
         self.input_plugin.report_progress = ir
+        if self.for_regex_wizard:
+            self.input_plugin.for_viewer = True
         with self.input_plugin:
             self.oeb = self.input_plugin(stream, self.opts,
                                         self.input_fmt, self.log,
@@ -1014,8 +1015,12 @@ OptionRecommendation(name='search_replace',
             if self.input_fmt in ('recipe', 'downloaded_recipe'):
                 self.opts_to_mi(self.user_metadata)
             if not hasattr(self.oeb, 'manifest'):
-                self.oeb = create_oebbook(self.log, self.oeb, self.opts,
-                        encoding=self.input_plugin.output_encoding)
+                self.oeb = create_oebbook(
+                    self.log, self.oeb, self.opts,
+                    encoding=self.input_plugin.output_encoding,
+                    for_regex_wizard=self.for_regex_wizard)
+            if self.for_regex_wizard:
+                return
             self.input_plugin.postprocess_book(self.oeb, self.opts, self.log)
             self.opts.is_image_collection = self.input_plugin.is_image_collection
             pr = CompositeProgressReporter(0.34, 0.67, self.ui_reporter)
@@ -1081,7 +1086,6 @@ OptionRecommendation(name='search_replace',
             self.dump_oeb(self.oeb, out_dir)
             self.log('Structured HTML written to:', out_dir)
 
-
         if self.opts.extra_css and os.path.exists(self.opts.extra_css):
             self.opts.extra_css = open(self.opts.extra_css, 'rb').read()
 
@@ -1161,13 +1165,20 @@ OptionRecommendation(name='search_replace',
         self.log(self.output_fmt.upper(), 'output written to', self.output)
         self.flush()
 
+# This has to be global as create_oebbook can be called from other locations
+# (for example in the html input plugin)
+regex_wizard_callback = None
+def set_regex_wizard_callback(f):
+    global regex_wizard_callback
+    regex_wizard_callback = f
+
 def create_oebbook(log, path_or_stream, opts, reader=None,
-        encoding='utf-8', populate=True):
+        encoding='utf-8', populate=True, for_regex_wizard=False):
     '''
     Create an OEBBook.
     '''
     from calibre.ebooks.oeb.base import OEBBook
-    html_preprocessor = HTMLPreProcessor(log, opts)
+    html_preprocessor = HTMLPreProcessor(log, opts, regex_wizard_callback=regex_wizard_callback)
     if not encoding:
         encoding = None
     oeb = OEBBook(log, html_preprocessor,
@@ -1182,3 +1193,4 @@ def create_oebbook(log, path_or_stream, opts, reader=None,
 
     reader()(oeb, path_or_stream)
     return oeb
+

src/calibre/ebooks/conversion/preprocess.py

@@ -14,7 +14,7 @@ SVG_NS       = 'http://www.w3.org/2000/svg'
 XLINK_NS     = 'http://www.w3.org/1999/xlink'
 
 convert_entities = functools.partial(entity_to_unicode,
-        result_exceptions = {
+        result_exceptions={
             u'<' : '&lt;',
             u'>' : '&gt;',
             u"'" : '&apos;',
@@ -144,9 +144,9 @@ class DocAnalysis(object):
         percent is the percentage of lines that should be in a single bucket to return true
         The majority of the lines will exist in 1-2 buckets in typical docs with hard line breaks
         '''
-        minLineLength=20 # Ignore lines under 20 chars (typical of spaces)
-        maxLineLength=1900 # Discard larger than this to stay in range
-        buckets=20 # Each line is divided into a bucket based on length
+        minLineLength=20  # Ignore lines under 20 chars (typical of spaces)
+        maxLineLength=1900  # Discard larger than this to stay in range
+        buckets=20  # Each line is divided into a bucket based on length
 
         #print "there are "+str(len(lines))+" lines"
         #max = 0
@@ -156,7 +156,7 @@ class DocAnalysis(object):
         #        max = l
         #print "max line found is "+str(max)
         # Build the line length histogram
-        hRaw = [ 0 for i in range(0,buckets) ]
+        hRaw = [0 for i in range(0,buckets)]
         for line in self.lines:
             l = len(line)
             if l > minLineLength and l < maxLineLength:
@@ -167,7 +167,7 @@ class DocAnalysis(object):
         # Normalize the histogram into percents
         totalLines = len(self.lines)
         if totalLines > 0:
-            h = [ float(count)/totalLines for count in hRaw ]
+            h = [float(count)/totalLines for count in hRaw]
         else:
             h = []
         #print "\nhRaw histogram lengths are: "+str(hRaw)
@@ -200,7 +200,7 @@ class Dehyphenator(object):
         # Add common suffixes to the regex below to increase the likelihood of a match -
         # don't add suffixes which are also complete words, such as 'able' or 'sex'
         # only remove if it's not already the point of hyphenation
-        self.suffix_string = "((ed)?ly|'?e?s||a?(t|s)?ion(s|al(ly)?)?|ings?|er|(i)?ous|(i|a)ty|(it)?ies|ive|gence|istic(ally)?|(e|a)nce|m?ents?|ism|ated|(e|u)ct(ed)?|ed|(i|ed)?ness|(e|a)ncy|ble|ier|al|ex|ian)$"
+        self.suffix_string = "((ed)?ly|'?e?s||a?(t|s)?ion(s|al(ly)?)?|ings?|er|(i)?ous|(i|a)ty|(it)?ies|ive|gence|istic(ally)?|(e|a)nce|m?ents?|ism|ated|(e|u)ct(ed)?|ed|(i|ed)?ness|(e|a)ncy|ble|ier|al|ex|ian)$"  # noqa
         self.suffixes = re.compile(r"^%s" % self.suffix_string, re.IGNORECASE)
         self.removesuffixes = re.compile(r"%s" % self.suffix_string, re.IGNORECASE)
         # remove prefixes if the prefix was not already the point of hyphenation
@@ -265,19 +265,18 @@ class Dehyphenator(object):
         self.html = html
         self.format = format
         if format == 'html':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags>(</span>)?\s*(</[iubp]>\s*){1,2}(?P<up2threeblanks><(p|div)[^>]*>\s*(<p[^>]*>\s*</p>\s*)?</(p|div)>\s+){0,3}\s*(<[iubp][^>]*>\s*){1,2}(<span[^>]*>)?)\s*(?P<secondpart>[\w\d]+)' % length)  # noqa
         elif format == 'pdf':
             intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<wraptags><p>|</[iub]>\s*<p>\s*<[iub]>)\s*(?P<secondpart>[\w\d]+)'% length)
         elif format == 'txt':
-            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)
+            intextmatch = re.compile(u'(?<=.{%i})(?P<firstpart>[^\W\-]+)(-|‐)(\u0020|\u0009)*(?P<wraptags>(\n(\u0020|\u0009)*)+)(?P<secondpart>[\w\d]+)'% length)  # noqa
         elif format == 'individual_words':
             intextmatch = re.compile(u'(?!<)(?P<firstpart>[^\W\-]+)(-|‐)\s*(?P<secondpart>\w+)(?![^<]*?>)')
         elif format == 'html_cleanup':
-            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')
+            intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)\s*(?=<)(?P<wraptags></span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*(?P<secondpart>[\w\d]+)')  # noqa
         elif format == 'txt_cleanup':
             intextmatch = re.compile(u'(?P<firstpart>[^\W\-]+)(-|‐)(?P<wraptags>\s+)(?P<secondpart>[\w\d]+)')
 
-
         html = intextmatch.sub(self.dehyphenate, html)
         return html
 
@@ -498,9 +497,11 @@ class HTMLPreProcessor(object):
                      (re.compile('<span[^><]*?id=subtitle[^><]*?>(.*?)</span>', re.IGNORECASE|re.DOTALL),
                       lambda match : '<h3 class="subtitle">%s</h3>'%(match.group(1),)),
                      ]
-    def __init__(self, log=None, extra_opts=None):
+    def __init__(self, log=None, extra_opts=None, regex_wizard_callback=None):
         self.log = log
         self.extra_opts = extra_opts
+        self.regex_wizard_callback = regex_wizard_callback
+        self.current_href = None
 
     def is_baen(self, src):
         return re.compile(r'<meta\s+name="Publisher"\s+content=".*?Baen.*?"',
@@ -581,12 +582,15 @@ class HTMLPreProcessor(object):
                 end_rules.append((re.compile(u'(?<=.{%i}[–—])\s*<p>\s*(?=[[a-z\d])' % length), lambda match: ''))
                 end_rules.append(
                     # Un wrap using punctuation
-                    (re.compile(u'(?<=.{%i}([a-zäëïöüàèìòùáćéíĺóŕńśúýâêîôûçąężıãõñæøþðßěľščťžňďřů,:)\IA\u00DF]|(?<!\&\w{4});))\s*(?P<ital></(i|b|u)>)?\s*(</p>\s*<p>\s*)+\s*(?=(<(i|b|u)>)?\s*[\w\d$(])' % length, re.UNICODE), wrap_lines),
+                    (re.compile(u'(?<=.{%i}([a-zäëïöüàèìòùáćéíĺóŕńśúýâêîôûçąężıãõñæøþðßěľščťžňďřů,:)\IA\u00DF]|(?<!\&\w{4});))\s*(?P<ital></(i|b|u)>)?\s*(</p>\s*<p>\s*)+\s*(?=(<(i|b|u)>)?\s*[\w\d$(])' % length, re.UNICODE), wrap_lines),  # noqa
                 )
 
         for rule in self.PREPROCESS + start_rules:
             html = rule[0].sub(rule[1], html)
 
+        if self.regex_wizard_callback is not None:
+            self.regex_wizard_callback(self.current_href, html)
+
         if get_preprocess_html:
             return html
 

src/calibre/ebooks/docx/block_styles.py

@@ -175,6 +175,20 @@ def read_shd(parent, dest):
         if val:
             ans = simple_color(val, auto='transparent')
     setattr(dest, 'background_color', ans)
+
+def read_numbering(parent, dest):
+    lvl = num_id = None
+    for np in XPath('./w:numPr')(parent):
+        for ilvl in XPath('./w:ilvl[@w:val]')(np):
+            try:
+                lvl = int(get(ilvl, 'w:val'))
+            except (ValueError, TypeError):
+                pass
+        for num in XPath('./w:numId[@w:val]')(np):
+            num_id = get(num, 'w:val')
+    val = (num_id, lvl) if num_id is not None or lvl is not None else inherit
+    setattr(dest, 'numbering', val)
+
 # }}}
 
 class ParagraphStyle(object):
@@ -194,6 +208,7 @@ class ParagraphStyle(object):
 
         # Misc.
         'text_indent', 'text_align', 'line_height', 'direction', 'background_color',
+        'numbering',
     )
 
     def __init__(self, pPr=None):
@@ -210,7 +225,7 @@ class ParagraphStyle(object):
             ):
                 setattr(self, p, binary_property(pPr, p))
 
-            for x in ('border', 'indent', 'justification', 'spacing', 'direction', 'shd'):
+            for x in ('border', 'indent', 'justification', 'spacing', 'direction', 'shd', 'numbering'):
                 f = globals()['read_%s' % x]
                 f(pPr, self)
 

src/calibre/ebooks/docx/dump.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import sys, os, shutil
+
+from lxml import etree
+
+from calibre import walk
+from calibre.utils.zipfile import ZipFile
+
+def dump(path):
+    dest = os.path.splitext(os.path.basename(path))[0]
+    dest += '_extracted'
+    if os.path.exists(dest):
+        shutil.rmtree(dest)
+    with ZipFile(path) as zf:
+        zf.extractall(dest)
+
+    for f in walk(dest):
+        if f.endswith('.xml'):
+            with open(f, 'r+b') as stream:
+                raw = stream.read()
+                root = etree.fromstring(raw)
+                stream.seek(0)
+                stream.truncate()
+                stream.write(etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=True))
+
+    print (path, 'dumped to', dest)
+
+if __name__ == '__main__':
+    dump(sys.argv[-1])
+

src/calibre/ebooks/docx/names.py

@@ -45,8 +45,13 @@ namespaces = {
     'dcterms': 'http://purl.org/dc/terms/'
 }
 
+xpath_cache = {}
+
 def XPath(expr):
-    return X(expr, namespaces=namespaces)
+    ans = xpath_cache.get(expr, None)
+    if ans is None:
+        xpath_cache[expr] = ans = X(expr, namespaces=namespaces)
+    return ans
 
 def is_tag(x, q):
     tag = getattr(x, 'tag', x)

src/calibre/ebooks/docx/numbering.py

@@ -6,6 +6,11 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
+import re
+from collections import Counter
+
+from lxml.html.builder import OL, UL, SPAN
+
 from calibre.ebooks.docx.block_styles import ParagraphStyle
 from calibre.ebooks.docx.char_styles import RunStyle
 from calibre.ebooks.docx.names import XPath, get
@@ -33,10 +38,26 @@ class Level(object):
         self.fmt = 'decimal'
         self.para_link = None
         self.paragraph_style = self.character_style = None
+        self.is_numbered = False
+        self.num_template = None
 
         if lvl is not None:
             self.read_from_xml(lvl)
 
+    def copy(self):
+        ans = Level()
+        for x in ('restart', 'start', 'fmt', 'para_link', 'paragraph_style', 'character_style', 'is_numbered', 'num_template'):
+            setattr(ans, x, getattr(self, x))
+        return ans
+
+    def format_template(self, counter, ilvl):
+        def sub(m):
+            x = int(m.group(1)) - 1
+            if x > ilvl or x not in counter:
+                return ''
+            return '%d' % (counter[x] - (0 if x == ilvl else 1))
+        return re.sub(r'%(\d+)', sub, self.num_template).rstrip() + '\xa0'
+
     def read_from_xml(self, lvl, override=False):
         for lr in XPath('./w:lvlRestart[@w:val]')(lvl):
             try:
@@ -57,9 +78,13 @@ class Level(object):
         for lr in XPath('./w:numFmt[@w:val]')(lvl):
             val = get(lr, 'w:val')
             if val == 'bullet':
+                self.is_numbered = False
                 self.fmt = {'\uf0a7':'square', 'o':'circle'}.get(lt, 'disc')
             else:
+                self.is_numbered = True
                 self.fmt = STYLE_MAP.get(val, 'decimal')
+                if lt and re.match(r'%\d+\.$', lt) is None:
+                    self.num_template = lt
 
         for lr in XPath('./w:pStyle[@w:val]')(lvl):
             self.para_link = get(lr, 'w:val')
@@ -78,12 +103,6 @@ class Level(object):
             else:
                 self.character_style.update(ps)
 
-    def copy(self):
-        ans = Level()
-        for x in ('restart', 'start', 'fmt', 'para_link', 'paragraph_style', 'character_style'):
-            setattr(ans, x, getattr(self, x))
-        return ans
-
 class NumberingDefinition(object):
 
     def __init__(self, parent=None):
@@ -107,6 +126,7 @@ class Numbering(object):
     def __init__(self):
         self.definitions = {}
         self.instances = {}
+        self.counters = {}
 
     def __call__(self, root, styles):
         ' Read all numbering style definitions '
@@ -131,6 +151,7 @@ class Numbering(object):
                     if alvl is None:
                         alvl = Level()
                     alvl.read_from_xml(lvl, override=True)
+            return nd
 
         next_pass = {}
         for n in XPath('./w:num[@w:numId]')(root):
@@ -154,3 +175,114 @@ class Numbering(object):
             if d is not None:
                 self.instances[num_id] = create_instance(n, d)
 
+        for num_id, d in self.instances.iteritems():
+            self.counters[num_id] = Counter({lvl:d.levels[lvl].start for lvl in d.levels})
+
+    def get_pstyle(self, num_id, style_id):
+        d = self.instances.get(num_id, None)
+        if d is not None:
+            for ilvl, lvl in d.levels.iteritems():
+                if lvl.para_link == style_id:
+                    return ilvl
+
+    def get_para_style(self, num_id, lvl):
+        d = self.instances.get(num_id, None)
+        if d is not None:
+            lvl = d.levels.get(lvl, None)
+            return getattr(lvl, 'paragraph_style', None)
+
+    def update_counter(self, counter, levelnum, levels):
+        counter[levelnum] += 1
+        for ilvl, lvl in levels.iteritems():
+            restart = lvl.restart
+            if (restart is None and ilvl == levelnum + 1) or restart == levelnum + 1:
+                counter[ilvl] = lvl.start
+
+    def apply_markup(self, items, body, styles, object_map):
+        for p, num_id, ilvl in items:
+            d = self.instances.get(num_id, None)
+            if d is not None:
+                lvl = d.levels.get(ilvl, None)
+                if lvl is not None:
+                    counter = self.counters[num_id]
+                    p.tag = 'li'
+                    p.set('value', '%s' % counter[ilvl])
+                    p.set('list-lvl', str(ilvl))
+                    p.set('list-id', num_id)
+                    if lvl.num_template is not None:
+                        val = lvl.format_template(counter, ilvl)
+                        p.set('list-template', val)
+                    self.update_counter(counter, ilvl, d.levels)
+
+        def commit(current_run):
+            if not current_run:
+                return
+            start = current_run[0]
+            parent = start.getparent()
+            idx = parent.index(start)
+
+            d = self.instances[start.get('list-id')]
+            ilvl = int(start.get('list-lvl'))
+            lvl = d.levels[ilvl]
+            lvlid = start.get('list-id') + start.get('list-lvl')
+            wrap = (OL if lvl.is_numbered else UL)('\n\t')
+            has_template = 'list-template' in start.attrib
+            if has_template:
+                wrap.set('lvlid', lvlid)
+            else:
+                wrap.set('class', styles.register({'list-style-type': lvl.fmt}, 'list'))
+            parent.insert(idx, wrap)
+            last_val = None
+            for child in current_run:
+                wrap.append(child)
+                child.tail = '\n\t'
+                if has_template:
+                    span = SPAN()
+                    span.text = child.text
+                    child.text = None
+                    for gc in child:
+                        span.append(gc)
+                    child.append(span)
+                    span = SPAN(child.get('list-template'))
+                    child.insert(0, span)
+                for attr in ('list-lvl', 'list-id', 'list-template'):
+                    child.attrib.pop(attr, None)
+                val = int(child.get('value'))
+                if last_val == val - 1 or wrap.tag == 'ul':
+                    child.attrib.pop('value')
+                last_val = val
+            current_run[-1].tail = '\n'
+            del current_run[:]
+
+        parents = set()
+        for child in body.iterdescendants('li'):
+            parents.add(child.getparent())
+
+        for parent in parents:
+            current_run = []
+            for child in parent:
+                if child.tag == 'li':
+                    if current_run:
+                        last = current_run[-1]
+                        if (last.get('list-id') , last.get('list-lvl')) != (child.get('list-id'), child.get('list-lvl')):
+                            commit(current_run)
+                    current_run.append(child)
+                else:
+                    commit(current_run)
+            commit(current_run)
+
+        for wrap in body.xpath('//ol[@lvlid]'):
+            wrap.attrib.pop('lvlid')
+            wrap.tag = 'div'
+            for i, li in enumerate(wrap.iterchildren('li')):
+                li.tag = 'div'
+                li.attrib.pop('value', None)
+                li.set('style', 'display:table-row')
+                obj = object_map[li]
+                bs = styles.para_cache[obj]
+                if i == 0:
+                    wrap.set('style', 'display:table; margin-left: %s' % (bs.css.get('margin-left', 0)))
+                bs.css.pop('margin-left', None)
+                for child in li:
+                    child.set('style', 'display:table-cell')
+

src/calibre/ebooks/docx/styles.py

@@ -198,8 +198,19 @@ class Styles(object):
                 if default_para.character_style is not None:
                     self.para_char_cache[p] = default_para.character_style
 
+            is_numbering = direct_formatting.numbering is not inherit
+            if is_numbering:
+                num_id, lvl = direct_formatting.numbering
+                if num_id is not None:
+                    p.set('calibre_num_id', '%s:%s' % (lvl, num_id))
+                if num_id is not None and lvl is not None:
+                    ps = self.numbering.get_para_style(num_id, lvl)
+                    if ps is not None:
+                        parent_styles.append(ps)
+
             for attr in ans.all_properties:
-                setattr(ans, attr, self.para_val(parent_styles, direct_formatting, attr))
+                if not (is_numbering and attr == 'text_indent'):  # skip text-indent for lists
+                    setattr(ans, attr, self.para_val(parent_styles, direct_formatting, attr))
         return ans
 
     def resolve_run(self, r):
@@ -244,10 +255,20 @@ class Styles(object):
             return self.resolve_run(obj)
 
     def resolve_numbering(self, numbering):
-        pass  # TODO: Implement this
+        # When a numPr element appears inside a paragraph style, the lvl info
+        # must be discarder and pStyle used instead.
+        self.numbering = numbering
+        for style in self:
+            ps = style.paragraph_style
+            if ps is not None and ps.numbering is not inherit:
+                lvl = numbering.get_pstyle(ps.numbering[0], style.style_id)
+                if lvl is None:
+                    ps.numbering = inherit
+                else:
+                    ps.numbering = (ps.numbering[0], lvl)
 
     def register(self, css, prefix):
-        h = hash(tuple(css.iteritems()))
+        h = hash(frozenset(css.iteritems()))
         ans, _ = self.classes.get(h, (None, None))
         if ans is None:
             self.counter[prefix] += 1
@@ -266,13 +287,15 @@ class Styles(object):
                 self.register(css, 'text')
 
     def class_name(self, css):
-        h = hash(tuple(css.iteritems()))
+        h = hash(frozenset(css.iteritems()))
         return self.classes.get(h, (None, None))[0]
 
     def generate_css(self):
         prefix = textwrap.dedent(
             '''\
-            p { margin: 0; padding: 0; text-indent: 1.5em }
+            p { text-indent: 1.5em }
+
+            ul, ol, p { margin: 0; padding: 0 }
             ''')
 
         ans = []

src/calibre/ebooks/docx/to_html.py

@@ -7,6 +7,7 @@ __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import sys, os, re
+from collections import OrderedDict
 
 from lxml import html
 from lxml.html.builder import (
@@ -36,7 +37,7 @@ class Convert(object):
         self.mi = self.docx.metadata
         self.body = BODY()
         self.styles = Styles()
-        self.object_map = {}
+        self.object_map = OrderedDict()
         self.html = HTML(
             HEAD(
                 META(charset='utf-8'),
@@ -72,6 +73,19 @@ class Convert(object):
                 pass  # TODO: Last section properties
             else:
                 self.log.debug('Unknown top-level tag: %s, ignoring' % barename(top_level.tag))
+
+        numbered = []
+        for html_obj, obj in self.object_map.iteritems():
+            raw = obj.get('calibre_num_id', None)
+            if raw is not None:
+                lvl, num_id = raw.partition(':')[0::2]
+                try:
+                    lvl = int(lvl)
+                except (TypeError, ValueError):
+                    lvl = 0
+                numbered.append((html_obj, num_id, lvl))
+        self.numbering.apply_markup(numbered, self.body, self.styles, self.object_map)
+
         if len(self.body) > 0:
             self.body.text = '\n\t'
             for child in self.body:
@@ -102,7 +116,7 @@ class Convert(object):
 
         nname = get_name(NUMBERING, 'numbering.xml')
         sname = get_name(STYLES, 'styles.xml')
-        numbering = Numbering()
+        numbering = self.numbering = Numbering()
 
         if sname is not None:
             try:
@@ -133,6 +147,7 @@ class Convert(object):
 
     def convert_p(self, p):
         dest = P()
+        self.object_map[dest] = p
         style = self.styles.resolve_paragraph(p)
         for run in XPath('descendant::w:r')(p):
             span = self.convert_run(run)
@@ -173,7 +188,6 @@ class Convert(object):
                 wrapper = self.wrap_elems(spans, SPAN())
                 wrapper.set('class', cls)
 
-        self.object_map[dest] = p
         return dest
 
     def wrap_elems(self, elems, wrapper):
@@ -188,7 +202,7 @@ class Convert(object):
 
     def convert_run(self, run):
         ans = SPAN()
-        ans.run = run
+        self.object_map[ans] = run
         text = Text(ans, 'text', [])
 
         for child in run:
@@ -224,7 +238,6 @@ class Convert(object):
             ans.tag = 'sub' if style.vert_align == 'subscript' else 'sup'
         if style.lang is not inherit:
             ans.lang = style.lang
-        self.object_map[ans] = run
         return ans
 
 if __name__ == '__main__':

src/calibre/ebooks/mobi/debug/mobi8.py

@@ -163,7 +163,8 @@ class MOBIFile(object):
             ext = 'dat'
             prefix = 'binary'
             suffix = ''
-            if sig in {b'HUFF', b'CDIC', b'INDX'}: continue
+            if sig in {b'HUFF', b'CDIC', b'INDX'}:
+                continue
             # TODO: Ignore CNCX records as well
             if sig == b'FONT':
                 font = read_font_record(rec.raw)
@@ -196,7 +197,6 @@ class MOBIFile(object):
             vals = list(index)[:-1] + [None, None, None, None]
             entry_map.append(Entry(*(vals[:12])))
 
-
         indexing_data = collect_indexing_data(entry_map, list(map(len,
             self.text_records)))
         self.indexing_data = [DOC + '\n' +textwrap.dedent('''\

src/calibre/ebooks/mobi/mobiml.py

@@ -16,7 +16,8 @@ from calibre.ebooks.oeb.transforms.flatcss import KeyMapper
 from calibre.utils.magick.draw import identify_data
 
 MBP_NS = 'http://mobipocket.com/ns/mbp'
-def MBP(name): return '{%s}%s' % (MBP_NS, name)
+def MBP(name):
+    return '{%s}%s' % (MBP_NS, name)
 
 MOBI_NSMAP = {None: XHTML_NS, 'mbp': MBP_NS}
 
@@ -413,7 +414,7 @@ class MobiMLizer(object):
                         # img sizes in units other than px
                         # See #7520 for test case
                         try:
-                            pixs = int(round(float(value) / \
+                            pixs = int(round(float(value) /
                                 (72./self.profile.dpi)))
                         except:
                             continue
@@ -488,8 +489,6 @@ class MobiMLizer(object):
         if elem.text:
             if istate.preserve:
                 text = elem.text
-            elif len(elem) > 0 and isspace(elem.text):
-                text = None
             else:
                 text = COLLAPSE.sub(' ', elem.text)
         valign = style['vertical-align']

src/calibre/ebooks/mobi/reader/headers.py

@@ -181,9 +181,9 @@ class BookHeader(object):
                 self.codec = 'cp1252' if not user_encoding else user_encoding
                 log.warn('Unknown codepage %d. Assuming %s' % (self.codepage,
                     self.codec))
-            # Some KF8 files have header length == 256 (generated by kindlegen
-            # 2.7?). See https://bugs.launchpad.net/bugs/1067310
-            max_header_length = 0x100
+            # Some KF8 files have header length == 264 (generated by kindlegen
+            # 2.9?). See https://bugs.launchpad.net/bugs/1179144
+            max_header_length = 0x108
 
             if (ident == 'TEXTREAD' or self.length < 0xE4 or
                     self.length > max_header_length or

src/calibre/ebooks/mobi/reader/markup.py

@@ -112,7 +112,7 @@ def update_flow_links(mobi8_reader, resource_map, log):
     url_css_index_pattern = re.compile(r'''kindle:flow:([0-9|A-V]+)\?mime=text/css[^\)]*''', re.IGNORECASE)
 
     for flow in mr.flows:
-        if flow is None: # 0th flow is None
+        if flow is None:  # 0th flow is None
             flows.append(flow)
             continue
 
@@ -330,7 +330,7 @@ def expand_mobi8_markup(mobi8_reader, resource_map, log):
     mobi8_reader.flows = flows
 
     # write out the parts and file flows
-    os.mkdir('text') # directory containing all parts
+    os.mkdir('text')  # directory containing all parts
     spine = []
     for i, part in enumerate(parts):
         pi = mobi8_reader.partinfo[i]

src/calibre/ebooks/oeb/base.py

@@ -871,6 +871,7 @@ class Manifest(object):
             orig_data = data
             fname = urlunquote(self.href)
             self.oeb.log.debug('Parsing', fname, '...')
+            self.oeb.html_preprocessor.current_href = self.href
             try:
                 data = parse_html(data, log=self.oeb.log,
                         decoder=self.oeb.decode,
@@ -1312,9 +1313,9 @@ class Guide(object):
                          ('notes', __('Notes')),
                          ('preface', __('Preface')),
                          ('text', __('Main Text'))]
-        TYPES = set(t for t, _ in _TYPES_TITLES)
+        TYPES = set(t for t, _ in _TYPES_TITLES)  # noqa
         TITLES = dict(_TYPES_TITLES)
-        ORDER = dict((t, i) for i, (t, _) in enumerate(_TYPES_TITLES))
+        ORDER = dict((t, i) for i, (t, _) in enumerate(_TYPES_TITLES))  # noqa
 
         def __init__(self, oeb, type, title, href):
             self.oeb = oeb

src/calibre/ebooks/oeb/iterator/__init__.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, re
+import sys, os, re
 
 from calibre.customize.ui import available_input_formats
 
@@ -26,17 +26,18 @@ def EbookIterator(*args, **kwargs):
     from calibre.ebooks.oeb.iterator.book import EbookIterator
     return EbookIterator(*args, **kwargs)
 
-def get_preprocess_html(path_to_ebook, output):
-    from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
-    iterator = EbookIterator(path_to_ebook)
-    iterator.__enter__(only_input_plugin=True, run_char_count=False,
-            read_anchor_map=False)
-    preprocessor = HTMLPreProcessor(None, False)
-    with open(output, 'wb') as out:
-        for path in iterator.spine:
-            with open(path, 'rb') as f:
-                html = f.read().decode('utf-8', 'replace')
-            html = preprocessor(html, get_preprocess_html=True)
+def get_preprocess_html(path_to_ebook, output=None):
+    from calibre.ebooks.conversion.plumber import set_regex_wizard_callback, Plumber
+    from calibre.utils.logging import DevNull
+    from calibre.ptempfile import TemporaryDirectory
+    raw = {}
+    set_regex_wizard_callback(raw.__setitem__)
+    with TemporaryDirectory('_regex_wiz') as tdir:
+        pl = Plumber(path_to_ebook, os.path.join(tdir, 'a.epub'), DevNull(), for_regex_wizard=True)
+        pl.run()
+        items = [raw[item.href] for item in pl.oeb.spine if item.href in raw]
+
+    with (sys.stdout if output is None else open(output, 'wb')) as out:
+        for html in items:
             out.write(html.encode('utf-8'))
             out.write(b'\n\n' + b'-'*80 + b'\n\n')
-

src/calibre/ebooks/oeb/iterator/book.py

@@ -25,7 +25,7 @@ from calibre.ebooks.oeb.transforms.cover import CoverManager
 from calibre.ebooks.oeb.iterator.spine import (SpineItem, create_indexing_data)
 from calibre.ebooks.oeb.iterator.bookmarks import BookmarksMixin
 
-TITLEPAGE = CoverManager.SVG_TEMPLATE.decode('utf-8').replace(\
+TITLEPAGE = CoverManager.SVG_TEMPLATE.decode('utf-8').replace(
         '__ar__', 'none').replace('__viewbox__', '0 0 600 800'
         ).replace('__width__', '600').replace('__height__', '800')
 

src/calibre/ebooks/oeb/parse_utils.py

@@ -44,8 +44,10 @@ META_XP = XPath('/h:html/h:head/h:meta[@http-equiv="Content-Type"]')
 
 def merge_multiple_html_heads_and_bodies(root, log=None):
     heads, bodies = xpath(root, '//h:head'), xpath(root, '//h:body')
-    if not (len(heads) > 1 or len(bodies) > 1): return root
-    for child in root: root.remove(child)
+    if not (len(heads) > 1 or len(bodies) > 1):
+        return root
+    for child in root:
+        root.remove(child)
     head = root.makeelement(XHTML('head'))
     body = root.makeelement(XHTML('body'))
     for h in heads:
@@ -88,7 +90,7 @@ def html5_parse(data, max_nesting_depth=100):
     # Check that the asinine HTML 5 algorithm did not result in a tree with
     # insane nesting depths
     for x in data.iterdescendants():
-        if isinstance(x.tag, basestring) and len(x) is 0: # Leaf node
+        if isinstance(x.tag, basestring) and len(x) is 0:  # Leaf node
             depth = node_depth(x)
             if depth > max_nesting_depth:
                 raise ValueError('html5lib resulted in a tree with nesting'
@@ -228,7 +230,7 @@ def parse_html(data, log=None, decoder=None, preprocessor=None,
     if idx > -1:
         pre = data[:idx]
         data = data[idx:]
-        if '<!DOCTYPE' in pre: # Handle user defined entities
+        if '<!DOCTYPE' in pre:  # Handle user defined entities
             user_entities = {}
             for match in re.finditer(r'<!ENTITY\s+(\S+)\s+([^>]+)', pre):
                 val = match.group(2)
@@ -368,8 +370,7 @@ def parse_html(data, log=None, decoder=None, preprocessor=None,
         meta.getparent().remove(meta)
     meta = etree.SubElement(head, XHTML('meta'),
         attrib={'http-equiv': 'Content-Type'})
-    meta.set('content', 'text/html; charset=utf-8') # Ensure content is second
-                                                    # attribute
+    meta.set('content', 'text/html; charset=utf-8')  # Ensure content is second attribute
 
     # Ensure has a <body/>
     if not xpath(data, '/h:html/h:body'):

src/calibre/ebooks/pdf/render/links.py

@@ -45,11 +45,15 @@ class Links(object):
             href, page, rect = link
             p, frag = href.partition('#')[0::2]
             try:
-                link = ((path, p, frag or None), self.pdf.get_pageref(page).obj, Array(rect))
+                pref = self.pdf.get_pageref(page).obj
             except IndexError:
-                self.log.warn('Unable to find page for link: %r, ignoring it' % link)
-                continue
-            self.links.append(link)
+                try:
+                    pref = self.pdf.get_pageref(page-1).obj
+                except IndexError:
+                    self.pdf.debug('Unable to find page for link: %r, ignoring it' % link)
+                    continue
+                self.pdf.debug('The link %s points to non-existent page, moving it one page back' % href)
+            self.links.append(((path, p, frag or None), pref, Array(rect)))
 
     def add_links(self):
         for link in self.links:

src/calibre/gui2/device.py

@@ -122,7 +122,8 @@ def device_name_for_plugboards(device_class):
 class DeviceManager(Thread): # {{{
 
     def __init__(self, connected_slot, job_manager, open_feedback_slot,
-            open_feedback_msg, allow_connect_slot, sleep_time=2):
+                 open_feedback_msg, allow_connect_slot,
+                 after_callback_feedback_slot, sleep_time=2):
         '''
         :sleep_time: Time to sleep between device probes in secs
         '''
@@ -150,6 +151,7 @@ class DeviceManager(Thread): # {{{
         self.ejected_devices  = set([])
         self.mount_connection_requests = Queue.Queue(0)
         self.open_feedback_slot = open_feedback_slot
+        self.after_callback_feedback_slot = after_callback_feedback_slot
         self.open_feedback_msg = open_feedback_msg
         self._device_information = None
         self.current_library_uuid = None
@@ -392,6 +394,10 @@ class DeviceManager(Thread): # {{{
                         self.device.set_progress_reporter(job.report_progress)
                     self.current_job.run()
                     self.current_job = None
+                    feedback = getattr(self.device, 'user_feedback_after_callback', None)
+                    if feedback is not None:
+                        self.device.user_feedback_after_callback = None
+                        self.after_callback_feedback_slot(feedback)
                 else:
                     break
             if do_sleep:
@@ -850,7 +856,7 @@ class DeviceMixin(object): # {{{
         self.device_manager = DeviceManager(FunctionDispatcher(self.device_detected),
                 self.job_manager, Dispatcher(self.status_bar.show_message),
                 Dispatcher(self.show_open_feedback),
-                FunctionDispatcher(self.allow_connect))
+                FunctionDispatcher(self.allow_connect), Dispatcher(self.after_callback_feedback))
         self.device_manager.start()
         self.device_manager.devices_initialized.wait()
         if tweaks['auto_connect_to_folder']:
@@ -862,6 +868,10 @@ class DeviceMixin(object): # {{{
                 name, show_copy_button=False,
                 override_icon=QIcon(icon))
 
+    def after_callback_feedback(self, feedback):
+        title, msg, det_msg = feedback
+        info_dialog(self, feedback['title'], feedback['msg'], det_msg=feedback['det_msg']).show()
+
     def debug_detection(self, done):
         self.debug_detection_callback = weakref.ref(done)
         self.device_manager.debug_detection(FunctionDispatcher(self.debug_detection_done))
@@ -1116,7 +1126,7 @@ class DeviceMixin(object): # {{{
             return
 
         dm = self.iactions['Remove Books'].delete_memory
-        if dm.has_key(job):
+        if job in dm:
             paths, model = dm.pop(job)
             self.device_manager.remove_books_from_metadata(paths,
                     self.booklists())
@@ -1141,7 +1151,7 @@ class DeviceMixin(object): # {{{
     def dispatch_sync_event(self, dest, delete, specific):
         rows = self.library_view.selectionModel().selectedRows()
         if not rows or len(rows) == 0:
-            error_dialog(self, _('No books'), _('No books')+' '+\
+            error_dialog(self, _('No books'), _('No books')+' '+
                     _('selected to send')).exec_()
             return
 
@@ -1160,7 +1170,7 @@ class DeviceMixin(object): # {{{
                 if fmts:
                     for f in fmts.split(','):
                         f = f.lower()
-                        if format_count.has_key(f):
+                        if f in format_count:
                             format_count[f] += 1
                         else:
                             format_count[f] = 1

src/calibre/gui2/dialogs/quickview.py

@@ -149,6 +149,9 @@ class Quickview(QDialog, Ui_Quickview):
         key = self.view.model().column_map[self.current_column]
         book_id = self.view.model().id(bv_row)
 
+        if self.current_book_id == book_id and self.current_key == key:
+            return
+
         # Only show items for categories
         if not self.db.field_metadata[key]['is_category']:
             if self.current_key is None:
@@ -203,8 +206,7 @@ class Quickview(QDialog, Ui_Quickview):
             sv = selected_item
         sv = sv.replace('"', r'\"')
         self.last_search = self.current_key+':"=' + sv + '"'
-        books = self.db.search_getting_ids(self.last_search,
-                                           self.db.data.search_restriction)
+        books = self.db.search(self.last_search, return_matches=True)
 
         self.books_table.setRowCount(len(books))
         self.books_label.setText(_('Books with selected item "{0}": {1}').

src/calibre/gui2/library/views.py

@@ -10,9 +10,9 @@ from functools import partial
 from future_builtins import map
 from collections import OrderedDict
 
-from PyQt4.Qt import (QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal,
-    QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication,
-    QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect)
+from PyQt4.Qt import (QTableView, Qt, QAbstractItemView, QMenu, pyqtSignal, QFont,
+    QModelIndex, QIcon, QItemSelection, QMimeData, QDrag, QApplication, QStyle,
+    QPoint, QPixmap, QUrl, QImage, QPainter, QColor, QRect, QHeaderView, QStyleOptionHeader)
 
 from calibre.gui2.library.delegates import (RatingDelegate, PubDateDelegate,
     TextDelegate, DateDelegate, CompleteDelegate, CcTextDelegate,
@@ -25,7 +25,55 @@ from calibre.gui2.library import DEFAULT_SORT
 from calibre.constants import filesystem_encoding
 from calibre import force_unicode
 
-class PreserveViewState(object): # {{{
+class HeaderView(QHeaderView):  # {{{
+
+    def __init__(self, *args):
+        QHeaderView.__init__(self, *args)
+        self.hover = -1
+        self.current_font = QFont(self.font())
+        self.current_font.setBold(True)
+        self.current_font.setItalic(True)
+
+    def event(self, e):
+        if e.type() in (e.HoverMove, e.HoverEnter):
+            self.hover = self.logicalIndexAt(e.pos())
+        elif e.type() in (e.Leave, e.HoverLeave):
+            self.hover = -1
+        return QHeaderView.event(self, e)
+
+    def paintSection(self, painter, rect, logical_index):
+        opt = QStyleOptionHeader()
+        self.initStyleOption(opt)
+        opt.rect = rect
+        opt.section = logical_index
+        opt.orientation = self.orientation()
+        opt.textAlignment = Qt.AlignHCenter | Qt.AlignVCenter
+        model = self.parent().model()
+        opt.text = model.headerData(logical_index, opt.orientation, Qt.DisplayRole).toString()
+        if self.isSortIndicatorShown() and self.sortIndicatorSection() == logical_index:
+            opt.sortIndicator = QStyleOptionHeader.SortDown if self.sortIndicatorOrder() == Qt.AscendingOrder else QStyleOptionHeader.SortUp
+        opt.text = opt.fontMetrics.elidedText(opt.text, Qt.ElideRight, rect.width() - 4)
+        if self.isEnabled():
+            opt.state |= QStyle.State_Enabled
+            if self.window().isActiveWindow():
+                opt.state |= QStyle.State_Active
+                if self.hover == logical_index:
+                    opt.state |= QStyle.State_MouseOver
+        sm = self.selectionModel()
+        if opt.orientation == Qt.Vertical:
+            if sm.isRowSelected(logical_index, QModelIndex()):
+                opt.state |= QStyle.State_Sunken
+
+        painter.save()
+        if (
+                (opt.orientation == Qt.Horizontal and sm.currentIndex().column() == logical_index) or
+                (opt.orientation == Qt.Vertical and sm.currentIndex().row() == logical_index)):
+            painter.setFont(self.current_font)
+        self.style().drawControl(QStyle.CE_Header, opt, painter, self)
+        painter.restore()
+# }}}
+
+class PreserveViewState(object):  # {{{
 
     '''
     Save the set of selected books at enter time. If at exit time there are no
@@ -72,13 +120,14 @@ class PreserveViewState(object): # {{{
             return {x:getattr(self, x) for x in ('selected_ids', 'current_id',
                 'vscroll', 'hscroll')}
         def fset(self, state):
-            for k, v in state.iteritems(): setattr(self, k, v)
+            for k, v in state.iteritems():
+                setattr(self, k, v)
             self.__exit__()
         return property(fget=fget, fset=fset)
 
 # }}}
 
-class BooksView(QTableView): # {{{
+class BooksView(QTableView):  # {{{
 
     files_dropped = pyqtSignal(object)
     add_column_signal = pyqtSignal()
@@ -90,6 +139,7 @@ class BooksView(QTableView): # {{{
 
     def __init__(self, parent, modelcls=BooksModel, use_edit_metadata_dialog=True):
         QTableView.__init__(self, parent)
+        self.setProperty('highlight_current_item', 150)
         self.row_sizing_done = False
 
         if not tweaks['horizontal_scrolling_per_column']:
@@ -152,12 +202,16 @@ class BooksView(QTableView): # {{{
         # {{{ Column Header setup
         self.can_add_columns = True
         self.was_restored = False
-        self.column_header = self.horizontalHeader()
+        self.column_header = HeaderView(Qt.Horizontal, self)
+        self.setHorizontalHeader(self.column_header)
         self.column_header.setMovable(True)
+        self.column_header.setClickable(True)
         self.column_header.sectionMoved.connect(self.save_state)
         self.column_header.setContextMenuPolicy(Qt.CustomContextMenu)
         self.column_header.customContextMenuRequested.connect(self.show_column_header_context_menu)
         self.column_header.sectionResized.connect(self.column_resized, Qt.QueuedConnection)
+        self.row_header = HeaderView(Qt.Vertical, self)
+        self.setVerticalHeader(self.row_header)
         # }}}
 
         self._model.database_changed.connect(self.database_changed)
@@ -235,7 +289,7 @@ class BooksView(QTableView): # {{{
                 ac.setCheckable(True)
                 ac.setChecked(True)
             if col not in ('ondevice', 'inlibrary') and \
-                    (not self.model().is_custom_column(col) or \
+                    (not self.model().is_custom_column(col) or
                     self.model().custom_columns[col]['datatype'] not in ('bool',
                         )):
                 m = self.column_header_context_menu.addMenu(
@@ -277,7 +331,6 @@ class BooksView(QTableView): # {{{
                         partial(self.column_header_context_handler,
                         action='show', column=col))
 
-
             self.column_header_context_menu.addSeparator()
             self.column_header_context_menu.addAction(
                     _('Shrink column if it is too wide to fit'),
@@ -366,7 +419,7 @@ class BooksView(QTableView): # {{{
         h = self.column_header
         cm = self.column_map
         state = {}
-        state['hidden_columns'] = [cm[i] for i in  range(h.count())
+        state['hidden_columns'] = [cm[i] for i in range(h.count())
                 if h.isSectionHidden(i) and cm[i] != 'ondevice']
         state['last_modified_injected'] = True
         state['languages_injected'] = True
@@ -514,7 +567,6 @@ class BooksView(QTableView): # {{{
                         db.prefs[name] = ans
         return ans
 
-
     def restore_state(self):
         old_state = self.get_old_state()
         if old_state is None:
@@ -837,7 +889,8 @@ class BooksView(QTableView): # {{{
         ids = frozenset(ids)
         m = self.model()
         for row in xrange(m.rowCount(QModelIndex())):
-            if len(row_map) >= len(ids): break
+            if len(row_map) >= len(ids):
+                break
             c = m.id(row)
             if c in ids:
                 row_map[c] = row
@@ -897,7 +950,8 @@ class BooksView(QTableView): # {{{
                 pass
             return None
         def fset(self, val):
-            if val is None: return
+            if val is None:
+                return
             m = self.model()
             for row in xrange(m.rowCount(QModelIndex())):
                 if m.id(row) == val:
@@ -919,7 +973,8 @@ class BooksView(QTableView): # {{{
         column = ci.column()
 
         for i in xrange(ci.row()+1, self.row_count()):
-            if i in selected_rows: continue
+            if i in selected_rows:
+                continue
             try:
                 return self.model().id(self.model().index(i, column))
             except:
@@ -927,7 +982,8 @@ class BooksView(QTableView): # {{{
 
         # No unselected rows after the current row, look before
         for i in xrange(ci.row()-1, -1, -1):
-            if i in selected_rows: continue
+            if i in selected_rows:
+                continue
             try:
                 return self.model().id(self.model().index(i, column))
             except:
@@ -975,7 +1031,7 @@ class BooksView(QTableView): # {{{
 
 # }}}
 
-class DeviceBooksView(BooksView): # {{{
+class DeviceBooksView(BooksView):  # {{{
 
     def __init__(self, parent):
         BooksView.__init__(self, parent, DeviceBooksModel,

src/qtcurve/style/qtcurve.cpp

@@ -964,6 +964,7 @@ Style::Style()
         itsAnimateStep(0),
         itsTitlebarHeight(0),
         calibre_icon_map(QHash<int,QString>()),
+        calibre_item_view_focus(0),
         is_kde_session(0),
         itsPos(-1, -1),
         itsHoverWidget(0L),
@@ -3696,6 +3697,9 @@ bool Style::event(QEvent *event) {
                 ++i;
             }
             return true;
+        } else if (e->propertyName() == QString("calibre_item_view_focus")) {
+            calibre_item_view_focus = property("calibre_item_view_focus").toInt();
+            return true;
         }
     }
     return BASE_STYLE::event(event);
@@ -4784,11 +4788,7 @@ void Style::drawPrimitive(PrimitiveElement element, const QStyleOption *option,
                 if(widget && ::qobject_cast<const QGroupBox *>(widget))
                    r2.adjust(0, 2, 0, 0);
 
-                // Added by Kovid so that the highlight does not cover the text
-                if(widget && ::qobject_cast<const QListView *>(widget))
-                   r2.adjust(0, 0, 0, 2);
-
-                if(FOCUS_STANDARD==opts.focus)
+                if(calibre_item_view_focus || FOCUS_STANDARD==opts.focus) // Changed by Kovid, as the underline focus does not work well in item views
                 {
                     // Taken from QWindowsStyle...
                     painter->save();
@@ -4803,10 +4803,11 @@ void Style::drawPrimitive(PrimitiveElement element, const QStyleOption *option,
                     painter->setBrush(QBrush(patternCol, Qt::Dense4Pattern));
                     painter->setBrushOrigin(r.topLeft());
                     painter->setPen(Qt::NoPen);
-                    painter->drawRect(r.left(), r.top(), r.width(), 1);    // Top
-                    painter->drawRect(r.left(), r.bottom(), r.width(), 1); // Bottom
-                    painter->drawRect(r.left(), r.top(), 1, r.height());   // Left
-                    painter->drawRect(r.right(), r.top(), 1, r.height());  // Right
+                    int fwidth = (calibre_item_view_focus > 1) ? 2 : 1;
+                    painter->drawRect(r.left(), r.top(), r.width(), fwidth);    // Top
+                    painter->drawRect(r.left(), r.bottom(), r.width(), fwidth); // Bottom
+                    painter->drawRect(r.left(), r.top(), fwidth, r.height());   // Left
+                    painter->drawRect(r.right(), r.top(), fwidth, r.height());  // Right
                     painter->restore();
                 }
                 else
@@ -5249,6 +5250,14 @@ void Style::drawPrimitive(PrimitiveElement element, const QStyleOption *option,
                 QColor color(hasCustomBackground && hasSolidBackground
                                 ? v4Opt->backgroundBrush.color()
                                 : palette.color(cg, QPalette::Highlight));
+                if (state & State_HasFocus && widget && widget->property("highlight_current_item").toBool()) {
+                    // Added by Kovid to highlight the current cell in the book list
+                    if (color.lightness() > 128)
+                        color = color.darker(widget->property("highlight_current_item").toInt());
+                    else
+                        color = color.lighter();
+                }
+                
                 bool   square((opts.square&SQUARE_LISTVIEW_SELECTION) &&
                               (/*(!widget && r.height()<=40 && r.width()>=48) || */
                                (widget && !widget->inherits("KFilePlacesView") &&

src/qtcurve/style/qtcurve.h

@@ -355,6 +355,7 @@ class Style : public QCommonStyle
     mutable QList<int>                 itsMdiButtons[2]; // 0=left, 1=right
     mutable int                        itsTitlebarHeight;
     QHash<int,QString>                 calibre_icon_map;
+    int                                calibre_item_view_focus;
     bool                               is_kde_session;
 
     // Required for Q3Header hover...