Merge from main branch

src/calibre/ebooks/lrf/input.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, textwrap
+import os, textwrap, sys
 from copy import deepcopy
 
 from lxml import etree
@@ -413,7 +413,12 @@ class LRFInput(InputFormatPlugin):
                 ('calibre', 'image-block'): image_block,
                 }
         transform = etree.XSLT(styledoc, extensions=extensions)
-        result = transform(doc)
+        try:
+            result = transform(doc)
+        except RuntimeError:
+            sys.setrecursionlimit(5000)
+            result = transform(doc)
+
         with open('content.opf', 'wb') as f:
             f.write(result)
         styles.write()

src/calibre/ebooks/metadata/sources/base.py

@@ -15,6 +15,7 @@ from calibre.customize import Plugin
 from calibre.utils.logging import ThreadSafeLog, FileStream
 from calibre.utils.config import JSONConfig
 from calibre.utils.titlecase import titlecase
+from calibre.ebooks.metadata import check_isbn
 
 msprefs = JSONConfig('metadata_sources.json')
 
@@ -236,6 +237,7 @@ class Source(Plugin):
             mi.title = fixcase(mi.title)
         mi.authors = list(map(fixcase, mi.authors))
         mi.tags = list(map(fixcase, mi.tags))
+        mi.isbn = check_isbn(mi.isbn)
 
     # }}}
 

src/calibre/ebooks/metadata/sources/identify.py

@@ -14,6 +14,7 @@ from io import BytesIO
 
 from calibre.customize.ui import metadata_plugins
 from calibre.ebooks.metadata.sources.base import create_log
+from calibre.ebooks.metadata.xisbn import xisbn
 
 # How long to wait for more results after first result is found
 WAIT_AFTER_FIRST_RESULT = 30 # seconds
@@ -120,7 +121,41 @@ def identify(log, abort, title=None, authors=None, identifiers=[], timeout=30):
     log('We have %d merged results, merging took: %.2f seconds' %
             (len(merged_results), time.time() - start_time))
 
+class ISBNMerge(object):
+
+    def __init__(self):
+        self.pools = {}
+
+    def isbn_in_pool(self, isbn):
+        if isbn:
+            for p in self.pools:
+                if isbn in p:
+                    return p
+        return None
+
+    def pool_has_result_from_same_source(self, pool, result):
+        results = self.pools[pool][1]
+        for r in results:
+            if r.identify_plugin is result.identify_plugin:
+                return True
+        return False
+
+    def add_result(self, result, isbn):
+        pool = self.isbn_in_pool(isbn)
+        if pool is None:
+            isbns, min_year = xisbn.get_isbn_pool(isbn)
+            if not isbns:
+                isbns = frozenset([isbn])
+            self.pool[isbns] = pool = (min_year, [])
+
+        if not self.pool_has_result_from_same_source(pool, result):
+            pool[1].append(result)
+
 def merge_identify_results(result_map, log):
-    pass
+    for plugin, results in result_map.iteritems():
+        for result in results:
+            isbn = result.isbn
+            if isbn:
+                isbns, min_year = xisbn.get_isbn_pool(isbn)
 
 

src/calibre/ebooks/metadata/xisbn.py

@@ -71,6 +71,20 @@ class xISBN(object):
                 ans.add(i)
         return ans
 
+    def get_isbn_pool(self, isbn):
+        data = self.get_data(isbn)
+        isbns = frozenset([x.get('isbn') for x in data if 'isbn' in x])
+        min_year = 100000
+        for x in data:
+            try:
+                year = int(x['year'])
+                if year < min_year:
+                    min_year = year
+            except:
+                continue
+        if min_year == 100000:
+            min_year = None
+        return isbns, min_year
 
 
 xisbn = xISBN()

src/calibre/ebooks/pdf/input.py

@@ -34,7 +34,7 @@ class PDFInput(InputFormatPlugin):
         from calibre.ebooks.pdf.reflow import PDFDocument
         if pdfreflow_err:
             raise RuntimeError('Failed to load pdfreflow: ' + pdfreflow_err)
-        pdfreflow.reflow(stream.read())
+        pdfreflow.reflow(stream.read(), 1, -1)
         xml = open('index.xml', 'rb').read()
         PDFDocument(xml, self.opts, self.log)
         return os.path.join(os.getcwd(), 'metadata.opf')

src/calibre/ebooks/pdf/main.cpp

@@ -24,13 +24,14 @@ extern "C" {
     pdfreflow_reflow(PyObject *self, PyObject *args) {
         char *pdfdata;
         Py_ssize_t size;
+        int first_page, last_page, num = 0;
 
-        if (!PyArg_ParseTuple(args, "s#", &pdfdata, &size))
+        if (!PyArg_ParseTuple(args, "s#ii", &pdfdata, &size, &first_page, &last_page))
             return NULL;
 
         try {
             Reflow reflow(pdfdata, static_cast<std::ifstream::pos_type>(size));
-            reflow.render();
+            num = reflow.render(first_page, last_page);
         } catch (std::exception &e) {
             PyErr_SetString(PyExc_RuntimeError, e.what()); return NULL;
         } catch (...) {
@@ -38,7 +39,7 @@ extern "C" {
                     "Unknown exception raised while rendering PDF"); return NULL;
         }
 
-        Py_RETURN_NONE;
+        return Py_BuildValue("i", num);
     }
 
     static PyObject *
@@ -166,8 +167,8 @@ extern "C" {
     static 
     PyMethodDef pdfreflow_methods[] = {
         {"reflow", pdfreflow_reflow, METH_VARARGS,
-        "reflow(pdf_data)\n\n"
+        "reflow(pdf_data, first_page, last_page)\n\n"
-                "Reflow the specified PDF."
+                "Reflow the specified PDF. Returns the number of pages in the PDF. If last_page is -1 renders to end of document."
         },
         {"get_metadata", pdfreflow_get_metadata, METH_VARARGS,
         "get_metadata(pdf_data, cover)\n\n"

src/calibre/ebooks/pdf/reflow.cpp

@@ -712,16 +712,18 @@ Reflow::Reflow(char *pdfdata, size_t sz) :
 
 }
 
-void
+int
-Reflow::render() {
+Reflow::render(int first_page, int last_page) {
 
     if (!this->doc->okToCopy()) 
         cout << "Warning, this document has the copy protection flag set, ignoring." << endl;
 
     globalParams->setTextEncoding(encoding);
 
-    int first_page = 1;
+    int doc_pages = doc->getNumPages();
-    int last_page = doc->getNumPages();
+    if (last_page < 1 or last_page > doc_pages) last_page = doc_pages;
+    if (first_page < 1) first_page = 1;
+    if (first_page > last_page) first_page = last_page;
 
     XMLOutputDev *xml_out = new XMLOutputDev(this->doc);
     doc->displayPages(xml_out, first_page, last_page,
@@ -733,9 +735,12 @@ Reflow::render() {
               false //Printing
     );
     
-    this->dump_outline();
+    if (last_page - first_page == doc_pages - 1)
+        this->dump_outline();
 
     delete xml_out;
+
+    return doc_pages;
 }
 
 void Reflow::dump_outline() {

src/calibre/ebooks/pdf/reflow.h

@@ -66,7 +66,7 @@ class Reflow {
         ~Reflow();
         
         /* Convert the PDF to XML. All files are output to the current directory */
-        void render();
+        int render(int first_page, int last_page);
 
         /* Get the PDF Info Dictionary */
         map<string, string> get_info();

src/calibre/gui2/actions/convert.py

@@ -51,7 +51,7 @@ class ConvertAction(InterfaceAction):
         self.queue_convert_jobs(jobs, changed, bad, rows, previous,
                 self.book_auto_converted, extra_job_args=[on_card])
 
-    def auto_convert_mail(self, to, fmts, delete_from_library, book_ids, format):
+    def auto_convert_mail(self, to, fmts, delete_from_library, book_ids, format, subject):
         previous = self.gui.library_view.currentIndex()
         rows = [x.row() for x in \
                 self.gui.library_view.selectionModel().selectedRows()]
@@ -59,7 +59,7 @@ class ConvertAction(InterfaceAction):
         if jobs == []: return
         self.queue_convert_jobs(jobs, changed, bad, rows, previous,
                 self.book_auto_converted_mail,
-                extra_job_args=[delete_from_library, to, fmts])
+                extra_job_args=[delete_from_library, to, fmts, subject])
 
     def auto_convert_news(self, book_ids, format):
         previous = self.gui.library_view.currentIndex()
@@ -145,9 +145,10 @@ class ConvertAction(InterfaceAction):
         self.gui.sync_to_device(on_card, False, specific_format=fmt, send_ids=[book_id], do_auto_convert=False)
 
     def book_auto_converted_mail(self, job):
-        temp_files, fmt, book_id, delete_from_library, to, fmts = self.conversion_jobs[job]
+        temp_files, fmt, book_id, delete_from_library, to, fmts, subject = self.conversion_jobs[job]
         self.book_converted(job)
-        self.gui.send_by_mail(to, fmts, delete_from_library, specific_format=fmt, send_ids=[book_id], do_auto_convert=False)
+        self.gui.send_by_mail(to, fmts, delete_from_library, subject=subject,
+                specific_format=fmt, send_ids=[book_id], do_auto_convert=False)
 
     def book_auto_converted_news(self, job):
         temp_files, fmt, book_id = self.conversion_jobs[job]

src/calibre/gui2/actions/device.py

@@ -82,7 +82,8 @@ class ShareConnMenu(QMenu): # {{{
             keys = sorted(opts.accounts.keys())
             for account in keys:
                 formats, auto, default = opts.accounts[account]
-                dest = 'mail:'+account+';'+formats
+                subject = opts.subjects.get(account, '')
+                dest = 'mail:'+account+';'+formats+';'+subject
                 action1 = DeviceAction(dest, False, False, I('mail.png'),
                         account)
                 action2 = DeviceAction(dest, True, False, I('mail.png'),

src/calibre/gui2/device.py

@@ -887,9 +887,14 @@ class DeviceMixin(object): # {{{
                 on_card = dest
             self.sync_to_device(on_card, delete, fmt)
         elif dest == 'mail':
-            to, fmts = sub_dest.split(';')
+            sub_dest_parts = sub_dest.split(';')
+            while len(sub_dest_parts) < 3:
+                sub_dest_parts.append('')
+            to = sub_dest_parts[0]
+            fmts = sub_dest_parts[1]
+            subject = ';'.join(sub_dest_parts[2:]) 
             fmts = [x.strip().lower() for x in fmts.split(',')]
-            self.send_by_mail(to, fmts, delete)
+            self.send_by_mail(to, fmts, delete, subject=subject)
 
     def cover_to_thumbnail(self, data):
         if self.device_manager.device and \

src/calibre/gui2/email.py

@@ -22,6 +22,7 @@ from calibre.customize.ui import available_input_formats, available_output_forma
 from calibre.ebooks.metadata import authors_to_string
 from calibre.constants import preferred_encoding
 from calibre.gui2 import config, Dispatcher, warning_dialog
+from calibre.library.save_to_disk import get_components
 from calibre.utils.config import tweaks
 
 class EmailJob(BaseJob): # {{{
@@ -210,7 +211,7 @@ class EmailMixin(object): # {{{
     def __init__(self):
         self.emailer = Emailer(self.job_manager)
 
-    def send_by_mail(self, to, fmts, delete_from_library, send_ids=None,
+    def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None,
             do_auto_convert=True, specific_format=None):
         ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids
         if not ids or len(ids) == 0:
@@ -239,7 +240,14 @@ class EmailMixin(object): # {{{
                 remove_ids.append(id)
                 jobnames.append(t)
                 attachments.append(f)
-                subjects.append(_('E-book:')+ ' '+t)
+                if not subject:
+                    subjects.append(_('E-book:')+ ' '+t)
+                else:
+                    components = get_components(subject, mi, id)
+                    if not components:
+                        components = [mi.title]
+                    subject = os.path.join(*components)
+                    subjects.append(subject)
                 a = authors_to_string(mi.authors if mi.authors else \
                         [_('Unknown')])
                 texts.append(_('Attached, you will find the e-book') + \
@@ -292,7 +300,7 @@ class EmailMixin(object): # {{{
                 if self.auto_convert_question(
                     _('Auto convert the following books before sending via '
                         'email?'), autos):
-                    self.iactions['Convert Books'].auto_convert_mail(to, fmts, delete_from_library, auto, format)
+                    self.iactions['Convert Books'].auto_convert_mail(to, fmts, delete_from_library, auto, format, subject)
 
         if bad:
             bad = '\n'.join('%s'%(i,) for i in bad)

src/calibre/gui2/library/models.py

@@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
 
 import shutil, functools, re, os, traceback
 from contextlib import closing
-from operator import attrgetter
 
 from PyQt4.Qt import QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, \
         QModelIndex, QVariant, QDate, QColor
@@ -18,7 +17,7 @@ from calibre.ebooks.metadata import fmt_sidx, authors_to_string, string_to_autho
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
-from calibre.utils.icu import sort_key, strcmp as icu_strcmp
+from calibre.utils.icu import sort_key
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
@@ -984,6 +983,21 @@ class OnDeviceSearch(SearchQueryParser): # {{{
 
 # }}}
 
+class DeviceDBSortKeyGen(object): # {{{
+
+    def __init__(self, attr, keyfunc, db):
+        self.attr = attr
+        self.db = db
+        self.keyfunc = keyfunc
+
+    def __call__(self, x):
+        try:
+            ans = self.keyfunc(getattr(self.db[x], self.attr))
+        except:
+            ans = None
+        return ans
+# }}}
+
 class DeviceBooksModel(BooksModel): # {{{
 
     booklist_dirtied = pyqtSignal()
@@ -1089,59 +1103,40 @@ class DeviceBooksModel(BooksModel): # {{{
 
     def sort(self, col, order, reset=True):
         descending = order != Qt.AscendingOrder
-        def strcmp(attr):
-            ag = attrgetter(attr)
-            def _strcmp(x, y):
-                x = ag(self.db[x])
-                y = ag(self.db[y])
-                if x == None:
-                    x = ''
-                if y == None:
-                    y = ''
-                return icu_strcmp(x.strip(), y.strip())
-            return _strcmp
-        def datecmp(x, y):
-            x = self.db[x].datetime
-            y = self.db[y].datetime
-            return cmp(dt_factory(x, assume_utc=True), dt_factory(y,
-                assume_utc=True))
-        def sizecmp(x, y):
-            x, y = int(self.db[x].size), int(self.db[y].size)
-            return cmp(x, y)
-        def tagscmp(x, y):
-            x = ','.join(sorted(getattr(self.db[x], 'device_collections', []),key=sort_key))
-            y = ','.join(sorted(getattr(self.db[y], 'device_collections', []),key=sort_key))
-            return cmp(x, y)
-        def libcmp(x, y):
-            x, y = self.db[x].in_library, self.db[y].in_library
-            return cmp(x, y)
-        def authorcmp(x, y):
-            ax = getattr(self.db[x], 'author_sort', None)
-            ay = getattr(self.db[y], 'author_sort', None)
-            if ax and ay:
-                x = ax
-                y = ay
-            else:
-                x, y = authors_to_string(self.db[x].authors), \
-                                authors_to_string(self.db[y].authors)
-            return cmp(x, y)
         cname = self.column_map[col]
-        fcmp = {
+        def author_key(x):
-                'title': strcmp('title_sorter'),
+            try:
-                'authors' : authorcmp,
+                ax = self.db[x].author_sort
-                'size' : sizecmp,
+                if not ax:
-                'timestamp': datecmp,
+                    raise Exception('')
-                'collections': tagscmp,
+            except:
-                'inlibrary': libcmp,
+                try:
+                    ax = authors_to_string(self.db[x].authors)
+                except:
+                    ax = ''
+            return ax
+
+        keygen = {
+                'title': ('title_sorter', lambda x: sort_key(x) if x else ''),
+                'authors' : author_key,
+                'size' : ('size', int),
+                'timestamp': ('datetime', functools.partial(dt_factory, assume_utc=True)),
+                'collections': ('device_collections', lambda x:sorted(x,
+                    key=sort_key)),
+                'inlibrary': ('in_library', lambda x: x),
                 }[cname]
-        self.map.sort(cmp=fcmp, reverse=descending)
+        keygen = keygen if callable(keygen) else DeviceDBSortKeyGen(
+            keygen[0], keygen[1], self.db)
+        self.map.sort(key=keygen, reverse=descending)
         if len(self.map) == len(self.db):
             self.sorted_map = list(self.map)
         else:
             self.sorted_map = list(range(len(self.db)))
-            self.sorted_map.sort(cmp=fcmp, reverse=descending)
+            self.sorted_map.sort(cmp=keygen, reverse=descending)
         self.sorted_on = (self.column_map[col], order)
         self.sort_history.insert(0, self.sorted_on)
+        if hasattr(keygen, 'db'):
+            keygen.db = None
         if reset:
             self.reset()
 

src/calibre/gui2/preferences/emailp.py

@@ -5,6 +5,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import textwrap
+
 from PyQt4.Qt import QAbstractTableModel, QVariant, QFont, Qt
 
 
@@ -17,25 +19,30 @@ from calibre.utils.smtp import config as smtp_prefs
 
 class EmailAccounts(QAbstractTableModel): # {{{
 
-    def __init__(self, accounts):
+    def __init__(self, accounts, subjects):
         QAbstractTableModel.__init__(self)
         self.accounts = accounts
+        self.subjects = subjects
         self.account_order = sorted(self.accounts.keys())
-        self.headers  = map(QVariant, [_('Email'), _('Formats'), _('Auto send')])
+        self.headers  = map(QVariant, [_('Email'), _('Formats'), _('Subject'), _('Auto send')])
         self.default_font = QFont()
         self.default_font.setBold(True)
         self.default_font = QVariant(self.default_font)
-        self.tooltips =[NONE] + map(QVariant,
+        self.tooltips =[NONE] + list(map(QVariant, map(textwrap.fill,
             [_('Formats to email. The first matching format will be sent.'),
+             _('Subject of the email to use when sending. When left blank '
+               'the title will be used for the subject. Also, the same '
+               'templates used for "Save to disk" such as {title} and '
+               '{author_sort} can be used here.'),
              '<p>'+_('If checked, downloaded news will be automatically '
                      'mailed <br>to this email address '
-                     '(provided it is in one of the listed formats).')])
+                     '(provided it is in one of the listed formats).')])))
 
     def rowCount(self, *args):
         return len(self.account_order)
 
     def columnCount(self, *args):
-        return 3
+        return len(self.headers)
 
     def headerData(self, section, orientation, role):
         if role == Qt.DisplayRole and orientation == Qt.Horizontal:
@@ -56,14 +63,16 @@ class EmailAccounts(QAbstractTableModel): # {{{
                 return QVariant(account)
             if col ==  1:
                 return QVariant(self.accounts[account][0])
+            if col == 2:
+                return QVariant(self.subjects.get(account, ''))
         if role == Qt.FontRole and self.accounts[account][2]:
             return self.default_font
-        if role == Qt.CheckStateRole and col == 2:
+        if role == Qt.CheckStateRole and col == 3:
             return QVariant(Qt.Checked if self.accounts[account][1] else Qt.Unchecked)
         return NONE
 
     def flags(self, index):
-        if index.column() == 2:
+        if index.column() == 3:
             return QAbstractTableModel.flags(self, index)|Qt.ItemIsUserCheckable
         else:
             return QAbstractTableModel.flags(self, index)|Qt.ItemIsEditable
@@ -73,8 +82,10 @@ class EmailAccounts(QAbstractTableModel): # {{{
             return False
         row, col = index.row(), index.column()
         account = self.account_order[row]
-        if col == 2:
+        if col == 3:
             self.accounts[account][1] ^= True
+        if col == 2:
+            self.subjects[account] = unicode(value.toString())
         elif col == 1:
             self.accounts[account][0] = unicode(value.toString()).upper()
         else:
@@ -143,7 +154,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.send_email_widget.initialize(self.preferred_to_address)
         self.send_email_widget.changed_signal.connect(self.changed_signal.emit)
         opts = self.send_email_widget.smtp_opts
-        self._email_accounts = EmailAccounts(opts.accounts)
+        self._email_accounts = EmailAccounts(opts.accounts, opts.subjects)
         self._email_accounts.dataChanged.connect(lambda x,y:
                 self.changed_signal.emit())
         self.email_view.setModel(self._email_accounts)
@@ -170,6 +181,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         if not self.send_email_widget.set_email_settings(to_set):
             raise AbortCommit('abort')
         self.proxy['accounts'] =  self._email_accounts.accounts
+        self.proxy['subjects'] = self._email_accounts.subjects
 
         return ConfigWidgetBase.commit(self)
 

src/calibre/utils/smtp.py

@@ -250,6 +250,7 @@ def config(defaults=None):
     c = Config('smtp',desc) if defaults is None else StringConfig(defaults,desc)
     c.add_opt('from_')
     c.add_opt('accounts', default={})
+    c.add_opt('subjects', default={})
     c.add_opt('relay_host')
     c.add_opt('relay_port', default=25)
     c.add_opt('relay_username')