Sync to trunk.

recipes/hbr_blogs.recipe

@@ -6,7 +6,7 @@ class HBR(BasicNewsRecipe):
     title = 'Harvard Business Review Blogs'
     description = 'To subscribe go to http://hbr.harvardbusiness.org'
     needs_subscription = True
-    __author__ = 'Kovid Goyal and Sujata Raman, enhanced by BrianG'
+    __author__ = 'Kovid Goyal, enhanced by BrianG'
     language = 'en'
     no_stylesheets = True
 

src/calibre/customize/builtins.py

@@ -603,10 +603,11 @@ from calibre.devices.eslick.driver import ESLICK, EBK52
 from calibre.devices.nuut2.driver import NUUT2
 from calibre.devices.iriver.driver import IRIVER_STORY
 from calibre.devices.binatone.driver import README
-from calibre.devices.hanvon.driver import N516, EB511, ALEX, AZBOOKA, THEBOOK
+from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK,
+        LIBREAIR)
 from calibre.devices.edge.driver import EDGE
-from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \
-        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER
+from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS,
+        SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER)
 from calibre.devices.sne.driver import SNE
 from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL,
         GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR,
@@ -716,7 +717,7 @@ plugins += [
     EB600,
     README,
     N516,
-    THEBOOK,
+    THEBOOK, LIBREAIR,
     EB511,
     ELONEX,
     TECLAST_K3,

src/calibre/devices/hanvon/driver.py

@@ -52,6 +52,18 @@ class THEBOOK(N516):
     EBOOK_DIR_MAIN = 'My books'
     WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGE'
 
+class LIBREAIR(N516):
+    name = 'Libre Air Driver'
+    gui_name = 'Libre Air'
+    description    = _('Communicate with the Libre Air reader.')
+    author         = 'Kovid Goyal'
+    FORMATS = ['epub', 'mobi', 'prc', 'fb2', 'rtf', 'txt', 'pdf']
+
+    BCD = [0x399]
+    VENDOR_NAME      = 'ALURATEK'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '_FILE-STOR_GADGET'
+    EBOOK_DIR_MAIN = 'Books'
+
 class ALEX(N516):
 
     name = 'Alex driver'

src/calibre/ebooks/epub/output.py

@@ -394,6 +394,13 @@ class EPUBOutput(OutputFormatPlugin):
             for tag in XPath('//h:img[@src]')(root):
                 tag.set('src', tag.get('src', '').replace('&', ''))
 
+            # ADE whimpers in fright when it encounters a <td> outside a
+            # <table>
+            in_table = XPath('ancestor::h:table')
+            for tag in XPath('//h:td|//h:tr|//h:th')(root):
+                if not in_table(tag):
+                    tag.tag = XHTML('div')
+
             special_chars = re.compile(u'[\u200b\u00ad]')
             for elem in root.iterdescendants():
                 if getattr(elem, 'text', False):
@@ -413,7 +420,7 @@ class EPUBOutput(OutputFormatPlugin):
                             rule.style.removeProperty('margin-left')
                             # padding-left breaks rendering in webkit and gecko
                             rule.style.removeProperty('padding-left')
-                # Change whitespace:pre to pre-line to accommodate readers that
+                # Change whitespace:pre to pre-wrap to accommodate readers that
                 # cannot scroll horizontally
                 for rule in stylesheet.data.cssRules.rulesOfType(CSSRule.STYLE_RULE):
                     style = rule.style

src/calibre/ebooks/mobi/mobiml.py

@@ -442,9 +442,12 @@ class MobiMLizer(object):
         if tag in TABLE_TAGS and self.ignore_tables:
             tag = 'span' if tag == 'td' else 'div'
 
-        # GR: Added 'width', 'border' and 'scope'
+        if tag == 'table':
+            css = style.cssdict()
+            if 'border' in css or 'border-width' in css:
+                elem.set('border', '1')
         if tag in TABLE_TAGS:
-            for attr in ('rowspan', 'colspan','width','border','scope'):
+            for attr in ('rowspan', 'colspan', 'width', 'border', 'scope'):
                 if attr in elem.attrib:
                     istate.attrib[attr] = elem.attrib[attr]
         if tag == 'q':

src/calibre/ebooks/mobi/reader.py

@@ -348,7 +348,6 @@ class MobiReader(object):
                 self.processed_html = self.remove_random_bytes(self.processed_html)
                 root = soupparser.fromstring(self.processed_html)
 
-
         if root.tag != 'html':
             self.log.warn('File does not have opening <html> tag')
             nroot = html.fromstring('<html><head></head><body></body></html>')

src/calibre/gui2/device.py

@@ -597,7 +597,7 @@ class DeviceMenu(QMenu): # {{{
 
     # }}}
 
-class DeviceSignals(QObject):
+class DeviceSignals(QObject): # {{{
     #: This signal is emitted once, after metadata is downloaded from the
     #: connected device.
     #: The sequence: gui.device_manager.is_device_connected will become True,
@@ -614,6 +614,7 @@ class DeviceSignals(QObject):
     device_connection_changed = pyqtSignal(object)
 
 device_signals = DeviceSignals()
+# }}}
 
 class DeviceMixin(object): # {{{
 

src/calibre/gui2/email.py

@@ -9,6 +9,7 @@ __docformat__ = 'restructuredtext en'
 import os, socket, time
 from binascii import unhexlify
 from functools import partial
+from threading import Thread
 from itertools import repeat
 
 from calibre.utils.smtp import (compose_mail, sendmail, extract_email_address,
@@ -22,9 +23,30 @@ from calibre.library.save_to_disk import get_components
 from calibre.utils.config import tweaks
 from calibre.gui2.threaded_jobs import ThreadedJob
 
+class Worker(Thread):
+
+    def __init__(self, func, args):
+        Thread.__init__(self)
+        self.daemon = True
+        self.exception = self.tb = None
+        self.func, self.args = func, args
+
+    def run(self):
+        #time.sleep(1000)
+        try:
+            self.func(*self.args)
+        except Exception as e:
+            import traceback
+            self.exception = e
+            self.tb = traceback.format_exc()
+        finally:
+            self.func = self.args = None
+
+
 class Sendmail(object):
 
     MAX_RETRIES = 1
+    TIMEOUT = 15 * 60 # seconds
 
     def __init__(self):
         self.calculate_rate_limit()
@@ -42,22 +64,32 @@ class Sendmail(object):
             abort=None, notifications=None):
 
         try_count = 0
-        while try_count <= self.MAX_RETRIES:
+        while True:
             if try_count > 0:
                 log('\nRetrying in %d seconds...\n' %
                         self.rate_limit)
-            try:
-                self.sendmail(attachment, aname, to, subject, text, log)
-                try_count = self.MAX_RETRIES
-                log('Email successfully sent')
-            except:
+            worker = Worker(self.sendmail,
+                    (attachment, aname, to, subject, text, log))
+            worker.start()
+            start_time = time.time()
+            while worker.is_alive():
+                worker.join(0.2)
                 if abort.is_set():
+                    log('Sending aborted by user')
                     return
-                if try_count == self.MAX_RETRIES:
-                    raise
-                log.exception('\nSending failed...\n')
-
+                if time.time() - start_time > self.TIMEOUT:
+                    log('Sending timed out')
+                    raise Exception(
+                            'Sending email %r to %r timed out, aborting'% (subject,
+                                to))
+            if worker.exception is None:
+                log('Email successfully sent')
+                return
+            log.error('\nSending failed...\n')
+            log.debug(worker.tb)
             try_count += 1
+            if try_count > self.MAX_RETRIES:
+                raise worker.exception
 
     def sendmail(self, attachment, aname, to, subject, text, log):
         while time.time() - self.last_send_time <= self.rate_limit:
@@ -90,7 +122,7 @@ def send_mails(jobnames, callback, attachments, to_s, subjects,
             attachments, to_s, subjects, texts, attachment_names):
         description = _('Email %s to %s') % (name, to)
         job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to,
-                subject, text), {}, callback, killable=False)
+                subject, text), {}, callback)
         job_manager.run_threaded_job(job)
 
 

src/calibre/gui2/jobs.py

@@ -457,7 +457,7 @@ class JobsDialog(QDialog, Ui_JobsDialog):
 
     def kill_job(self, *args):
         if question_dialog(self, _('Are you sure?'), _('Do you really want to stop the selected job?')):
-            for index in self.jobs_view.selectedIndexes():
+            for index in self.jobs_view.selectionModel().selectedRows():
                 row = index.row()
                 self.model.kill_job(row, self)
 

src/calibre/gui2/library/models.py

@@ -1110,6 +1110,8 @@ class DeviceBooksModel(BooksModel): # {{{
         if self.last_search:
             self.searched.emit(True)
 
+    def research(self, reset=True):
+        self.search(self.last_search, reset)
 
     def sort(self, col, order, reset=True):
         descending = order != Qt.AscendingOrder
@@ -1171,6 +1173,8 @@ class DeviceBooksModel(BooksModel): # {{{
         self.custom_columns = {}
         self.db = db
         self.map = list(range(0, len(db)))
+        self.research(reset=False)
+        self.resort()
 
     def cover(self, row):
         item = self.db[self.map[row]]
@@ -1319,8 +1323,6 @@ class DeviceBooksModel(BooksModel): # {{{
             ans = Qt.AlignVCenter | ALIGNMENT_MAP[self.alignment_map.get(cname,
                 'left')]
             return QVariant(ans)
-
-
         return NONE
 
     def headerData(self, section, orientation, role):

src/calibre/gui2/library/views.py

@@ -48,7 +48,7 @@ class BooksView(QTableView): # {{{
     files_dropped = pyqtSignal(object)
     add_column_signal = pyqtSignal()
 
-    def __init__(self, parent, modelcls=BooksModel):
+    def __init__(self, parent, modelcls=BooksModel, use_edit_metadata_dialog=True):
         QTableView.__init__(self, parent)
 
         self.setEditTriggers(self.EditKeyPressed)
@@ -60,8 +60,12 @@ class BooksView(QTableView): # {{{
         elif tweaks['doubleclick_on_library_view'] == 'edit_metadata':
             # Must not enable single-click to edit, or the field will remain
             # open in edit mode underneath the edit metadata dialog
+            if use_edit_metadata_dialog:
                 self.doubleClicked.connect(
-                        partial(parent.iactions['Edit Metadata'].edit_metadata, checked=False))
+                        partial(parent.iactions['Edit Metadata'].edit_metadata,
+                                checked=False))
+            else:
+                self.setEditTriggers(self.DoubleClicked|self.editTriggers())
 
         self.drag_allowed = True
         self.setDragEnabled(True)
@@ -792,7 +796,8 @@ class BooksView(QTableView): # {{{
 class DeviceBooksView(BooksView): # {{{
 
     def __init__(self, parent):
-        BooksView.__init__(self, parent, DeviceBooksModel)
+        BooksView.__init__(self, parent, DeviceBooksModel,
+                           use_edit_metadata_dialog=False)
         self.can_add_columns = False
         self.columns_resized = False
         self.resize_on_select = False

src/calibre/gui2/preferences/coloring.py

@@ -462,11 +462,11 @@ class EditRules(QWidget): # {{{
         self.l = l = QGridLayout(self)
         self.setLayout(l)
 
-        self.l1 = l1 = QLabel(_(
+        self.l1 = l1 = QLabel('<p>'+_(
             'You can control the color of columns in the'
             ' book list by creating "rules" that tell calibre'
             ' what color to use. Click the Add Rule button below'
-            ' to get started. You can change an existing rule by double'
+            ' to get started.<p>You can <b>change an existing rule</b> by double'
             ' clicking it.'))
         l1.setWordWrap(True)
         l.addWidget(l1, 0, 0, 1, 2)

src/calibre/gui2/tag_view.py

@@ -610,7 +610,7 @@ class TagTreeItem(object): # {{{
             self.temporary = temporary
             self.tag = Tag(data, category=category_key,
                    is_editable=category_key not in ['news', 'search', 'identifiers'],
-                   is_searchable=category_key not in ['news', 'search'])
+                   is_searchable=category_key not in ['search'])
 
         elif self.type == self.TAG:
             self.icon_state_map[0] = QVariant(data.icon)
@@ -1642,6 +1642,12 @@ class TagsModel(QAbstractItemModel): # {{{
 
         for node in self.category_nodes:
             if node.tag.state:
+                if node.category_key == "news":
+                    if node_searches[node.tag.state] == 'true':
+                        ans.append('tags:=news')
+                    else:
+                        ans.append('( not tags:=news )')
+                else:
                     ans.append('%s:%s'%(node.category_key, node_searches[node.tag.state]))
 
             key = node.category_key

src/calibre/library/database2.py

@@ -26,7 +26,7 @@ from calibre.library.custom_columns import CustomColumns
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.library.prefs import DBPrefs
 from calibre.ebooks.metadata.book.base import Metadata
-from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
+from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
@@ -188,8 +188,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         apply_default_prefs = not os.path.exists(self.dbpath)
         self.connect()
 
-        self.is_case_sensitive = not iswindows and not isosx and \
-            not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
+        self.is_case_sensitive = (not iswindows and
+            not os.path.exists(self.dbpath.replace('metadata.db',
+                'MeTAdAtA.dB')))
         SchemaUpgrade.__init__(self)
         # Guarantee that the library_id is set
         self.library_id
@@ -606,7 +607,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 f = self.format(id, format, index_is_id=True, as_file=True)
                 if f is None:
                     continue
-                with tempfile.SpooledTemporaryFile(max_size=100*(1024**2)) as stream:
+                with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream:
                     with f:
                         shutil.copyfileobj(f, stream)
                     stream.seek(0)

src/calibre/manual/conversion.rst

@@ -189,7 +189,7 @@ Extra CSS
 
 This option allows you to specify arbitrary CSS that will be applied to all HTML files in the
 input. This CSS is applied with very high priority and so should override most CSS present in
-the input document itself. You can use this setting to fine tune the presentation/layout of your
+the **input document** itself. You can use this setting to fine tune the presentation/layout of your
 document. For example, if you want all paragraphs of class `endnote` to be right aligned, just 
 add::
     
@@ -200,7 +200,8 @@ or if you want to change the indentation of all paragraphs::
     p { text-indent: 5mm; }
 
 :guilabel:`Extra CSS` is a very powerful option, but you do need an understanding of how CSS works
-to use it to its full potential.
+to use it to its full potential. You can use the debug pipeline option described above to see what
+CSS is present in your input document.
 
 Miscellaneous
 ~~~~~~~~~~~~~~

src/calibre/manual/template_lang.rst

@@ -417,7 +417,7 @@ You might find the following tips useful.
 
     * Create a custom composite column to test templates. Once you have the column, you can change its template simply by double-clicking on the column. Hide the column when you are not testing.
     * Templates can use other templates by referencing a composite custom column.
-    * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{null}``. This template will always evaluate to an empty string.
+    * In a plugboard, you can set a field to empty (or whatever is equivalent to empty) by using the special template ``{}``. This template will always evaluate to an empty string.
     * The technique described above to show numbers even if they have a zero value works with the standard field series_index.
 
 .. toctree::

src/calibre/web/feeds/__init__.py

@@ -317,6 +317,9 @@ def feed_from_xml(raw_xml, title=None, oldest_article=7,
                   max_articles_per_feed=100,
                   get_article_url=lambda item: item.get('link', None),
                   log=default_log):
+    # Handle unclosed escaped entities. They trip up feedparser and HBR for one
+    # generates them
+    raw_xml = re.sub(r'(&#\d+)([^0-9;])', r'\1;\2', raw_xml)
     feed = parse(raw_xml)
     pfeed = Feed(get_article_url=get_article_url, log=log)
     pfeed.populate_from_feed(feed, title=title,

src/calibre/web/feeds/news.py

@@ -13,8 +13,8 @@ from functools import partial
 from contextlib import nested, closing
 
 
-from calibre import browser, __appname__, iswindows, \
-                    strftime, preferred_encoding, as_unicode
+from calibre import (browser, __appname__, iswindows,
+                    strftime, preferred_encoding, as_unicode)
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre import entity_to_unicode

src/calibre/web/feeds/recipes/collection.py

@@ -101,6 +101,7 @@ def get_custom_recipe_collection(*args):
             if recipe_class is not None:
                 rmap['custom:%s'%id_] = recipe_class
         except:
+            print 'Failed to load recipe from: %r'%fname
             import traceback
             traceback.print_exc()
             continue